2024-11-25 12:57:22,133 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-25 12:57:22,158 main DEBUG Took 0.017829 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-25 12:57:22,158 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-25 12:57:22,159 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-25 12:57:22,162 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-25 12:57:22,165 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 12:57:22,189 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-25 12:57:22,214 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 12:57:22,216 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 12:57:22,217 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 12:57:22,217 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 12:57:22,218 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 12:57:22,218 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 12:57:22,220 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 12:57:22,221 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 12:57:22,222 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 12:57:22,222 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 12:57:22,224 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 12:57:22,225 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 12:57:22,226 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 12:57:22,227 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 12:57:22,228 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 12:57:22,229 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 12:57:22,231 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 12:57:22,231 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 12:57:22,232 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 12:57:22,232 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 12:57:22,233 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 12:57:22,233 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 12:57:22,233 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 12:57:22,234 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 12:57:22,234 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 12:57:22,235 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-25 12:57:22,236 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 12:57:22,238 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-25 12:57:22,242 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-25 12:57:22,242 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-25 12:57:22,244 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-25 12:57:22,244 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-25 12:57:22,255 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-25 12:57:22,258 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-25 12:57:22,260 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-25 12:57:22,261 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-25 12:57:22,261 main DEBUG createAppenders(={Console}) 2024-11-25 12:57:22,262 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-25 12:57:22,263 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-25 12:57:22,263 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-25 12:57:22,264 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-25 12:57:22,264 main DEBUG OutputStream closed 2024-11-25 12:57:22,264 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-25 12:57:22,266 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-25 12:57:22,266 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-25 12:57:22,393 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-25 12:57:22,395 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-25 12:57:22,401 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-25 12:57:22,402 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-25 12:57:22,403 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-25 12:57:22,404 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-25 12:57:22,404 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-25 12:57:22,405 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-25 12:57:22,405 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-25 12:57:22,405 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-25 12:57:22,406 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-25 12:57:22,406 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-25 12:57:22,407 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-25 12:57:22,407 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-25 12:57:22,408 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-25 12:57:22,408 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-25 12:57:22,408 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-25 12:57:22,409 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-25 12:57:22,412 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-25 12:57:22,412 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-25 12:57:22,413 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-25 12:57:22,414 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-25T12:57:22,736 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2bb80d27-3d57-c947-53d6-c4472333a06b 2024-11-25 12:57:22,738 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-25 12:57:22,739 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-25T12:57:22,748 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-11-25T12:57:22,774 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-25T12:57:22,792 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2bb80d27-3d57-c947-53d6-c4472333a06b/cluster_8b8d8747-9aba-2792-61f4-67cb1dbecf7b, deleteOnExit=true 2024-11-25T12:57:22,792 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-25T12:57:22,793 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2bb80d27-3d57-c947-53d6-c4472333a06b/test.cache.data in system properties and HBase conf 2024-11-25T12:57:22,794 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2bb80d27-3d57-c947-53d6-c4472333a06b/hadoop.tmp.dir in system properties and HBase conf 2024-11-25T12:57:22,795 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2bb80d27-3d57-c947-53d6-c4472333a06b/hadoop.log.dir in system properties and HBase conf 2024-11-25T12:57:22,796 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2bb80d27-3d57-c947-53d6-c4472333a06b/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-25T12:57:22,797 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2bb80d27-3d57-c947-53d6-c4472333a06b/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-25T12:57:22,797 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-25T12:57:22,906 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-25T12:57:23,022 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T12:57:23,027 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2bb80d27-3d57-c947-53d6-c4472333a06b/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-25T12:57:23,028 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2bb80d27-3d57-c947-53d6-c4472333a06b/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-25T12:57:23,029 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2bb80d27-3d57-c947-53d6-c4472333a06b/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-25T12:57:23,030 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2bb80d27-3d57-c947-53d6-c4472333a06b/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-25T12:57:23,031 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2bb80d27-3d57-c947-53d6-c4472333a06b/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-25T12:57:23,031 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2bb80d27-3d57-c947-53d6-c4472333a06b/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-25T12:57:23,032 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2bb80d27-3d57-c947-53d6-c4472333a06b/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-25T12:57:23,032 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2bb80d27-3d57-c947-53d6-c4472333a06b/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-25T12:57:23,033 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2bb80d27-3d57-c947-53d6-c4472333a06b/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-25T12:57:23,033 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2bb80d27-3d57-c947-53d6-c4472333a06b/nfs.dump.dir in system properties and HBase conf 2024-11-25T12:57:23,034 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2bb80d27-3d57-c947-53d6-c4472333a06b/java.io.tmpdir in system properties and HBase conf 2024-11-25T12:57:23,035 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2bb80d27-3d57-c947-53d6-c4472333a06b/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-25T12:57:23,035 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2bb80d27-3d57-c947-53d6-c4472333a06b/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-25T12:57:23,036 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2bb80d27-3d57-c947-53d6-c4472333a06b/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-25T12:57:24,367 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-25T12:57:24,448 INFO [Time-limited test {}] log.Log(170): Logging initialized @3531ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-25T12:57:24,563 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-25T12:57:24,687 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-25T12:57:24,780 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-25T12:57:24,781 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-25T12:57:24,784 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-25T12:57:24,806 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-25T12:57:24,841 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2bb80d27-3d57-c947-53d6-c4472333a06b/hadoop.log.dir/,AVAILABLE} 2024-11-25T12:57:24,844 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-25T12:57:25,130 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2bb80d27-3d57-c947-53d6-c4472333a06b/java.io.tmpdir/jetty-localhost-35027-hadoop-hdfs-3_4_1-tests_jar-_-any-11507404990227372225/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-25T12:57:25,136 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:35027} 2024-11-25T12:57:25,137 INFO [Time-limited test {}] server.Server(415): Started @4221ms 2024-11-25T12:57:25,717 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-25T12:57:25,728 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-25T12:57:25,733 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-25T12:57:25,733 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-25T12:57:25,734 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-25T12:57:25,735 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@42946cd0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2bb80d27-3d57-c947-53d6-c4472333a06b/hadoop.log.dir/,AVAILABLE} 2024-11-25T12:57:25,736 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-25T12:57:25,872 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7acb84b5{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2bb80d27-3d57-c947-53d6-c4472333a06b/java.io.tmpdir/jetty-localhost-37437-hadoop-hdfs-3_4_1-tests_jar-_-any-2723880180739968866/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-25T12:57:25,873 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6e1ebae6{HTTP/1.1, (http/1.1)}{localhost:37437} 2024-11-25T12:57:25,873 INFO [Time-limited test {}] server.Server(415): Started @4958ms 2024-11-25T12:57:25,944 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-25T12:57:27,110 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2bb80d27-3d57-c947-53d6-c4472333a06b/cluster_8b8d8747-9aba-2792-61f4-67cb1dbecf7b/data/data2/current/BP-2055677215-172.17.0.3-1732539443838/current, will proceed with Du for space computation calculation, 2024-11-25T12:57:27,110 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2bb80d27-3d57-c947-53d6-c4472333a06b/cluster_8b8d8747-9aba-2792-61f4-67cb1dbecf7b/data/data1/current/BP-2055677215-172.17.0.3-1732539443838/current, will proceed with Du for space computation calculation, 2024-11-25T12:57:27,217 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-25T12:57:27,282 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9f503a2f154b6d0a with lease ID 0x4dbeedda4a875d6d: Processing first storage report for DS-012750d6-b62b-46ef-b486-505acfd2e005 from datanode DatanodeRegistration(127.0.0.1:38683, datanodeUuid=5b24ef01-cddf-4ceb-adbc-7bc07f6c930d, infoPort=37633, infoSecurePort=0, ipcPort=36259, storageInfo=lv=-57;cid=testClusterID;nsid=1153895036;c=1732539443838) 2024-11-25T12:57:27,283 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9f503a2f154b6d0a with lease ID 0x4dbeedda4a875d6d: from storage DS-012750d6-b62b-46ef-b486-505acfd2e005 node DatanodeRegistration(127.0.0.1:38683, datanodeUuid=5b24ef01-cddf-4ceb-adbc-7bc07f6c930d, infoPort=37633, infoSecurePort=0, ipcPort=36259, storageInfo=lv=-57;cid=testClusterID;nsid=1153895036;c=1732539443838), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-25T12:57:27,284 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9f503a2f154b6d0a with lease ID 0x4dbeedda4a875d6d: Processing first storage report for DS-564187ec-a4ff-4643-a002-2180348be425 from datanode DatanodeRegistration(127.0.0.1:38683, datanodeUuid=5b24ef01-cddf-4ceb-adbc-7bc07f6c930d, infoPort=37633, infoSecurePort=0, ipcPort=36259, storageInfo=lv=-57;cid=testClusterID;nsid=1153895036;c=1732539443838) 2024-11-25T12:57:27,284 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9f503a2f154b6d0a with lease ID 0x4dbeedda4a875d6d: from storage DS-564187ec-a4ff-4643-a002-2180348be425 node DatanodeRegistration(127.0.0.1:38683, datanodeUuid=5b24ef01-cddf-4ceb-adbc-7bc07f6c930d, infoPort=37633, infoSecurePort=0, ipcPort=36259, storageInfo=lv=-57;cid=testClusterID;nsid=1153895036;c=1732539443838), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-25T12:57:27,358 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2bb80d27-3d57-c947-53d6-c4472333a06b 2024-11-25T12:57:27,525 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2bb80d27-3d57-c947-53d6-c4472333a06b/cluster_8b8d8747-9aba-2792-61f4-67cb1dbecf7b/zookeeper_0, clientPort=60142, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2bb80d27-3d57-c947-53d6-c4472333a06b/cluster_8b8d8747-9aba-2792-61f4-67cb1dbecf7b/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2bb80d27-3d57-c947-53d6-c4472333a06b/cluster_8b8d8747-9aba-2792-61f4-67cb1dbecf7b/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-25T12:57:27,537 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=60142 2024-11-25T12:57:27,550 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T12:57:27,553 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T12:57:27,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741825_1001 (size=7) 2024-11-25T12:57:28,174 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a with version=8 2024-11-25T12:57:28,175 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/hbase-staging 2024-11-25T12:57:28,284 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-25T12:57:28,576 INFO [Time-limited test {}] client.ConnectionUtils(128): master/1ff8009e463e:0 server-side Connection retries=45 2024-11-25T12:57:28,585 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T12:57:28,586 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-25T12:57:28,591 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-25T12:57:28,591 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T12:57:28,591 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-25T12:57:28,743 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-25T12:57:28,811 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-25T12:57:28,820 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-25T12:57:28,824 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-25T12:57:28,847 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 85120 (auto-detected) 2024-11-25T12:57:28,848 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-25T12:57:28,869 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:46289 2024-11-25T12:57:28,890 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:46289 connecting to ZooKeeper ensemble=127.0.0.1:60142 2024-11-25T12:57:28,971 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:462890x0, quorum=127.0.0.1:60142, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-25T12:57:28,974 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:46289-0x101735baab60000 connected 2024-11-25T12:57:29,097 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T12:57:29,101 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T12:57:29,115 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46289-0x101735baab60000, quorum=127.0.0.1:60142, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T12:57:29,119 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a, hbase.cluster.distributed=false 2024-11-25T12:57:29,146 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46289-0x101735baab60000, quorum=127.0.0.1:60142, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-25T12:57:29,151 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46289 2024-11-25T12:57:29,151 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46289 2024-11-25T12:57:29,154 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46289 2024-11-25T12:57:29,155 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46289 2024-11-25T12:57:29,156 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46289 2024-11-25T12:57:29,295 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/1ff8009e463e:0 server-side Connection retries=45 2024-11-25T12:57:29,297 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T12:57:29,297 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-25T12:57:29,297 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-25T12:57:29,298 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T12:57:29,298 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-25T12:57:29,301 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-25T12:57:29,305 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-25T12:57:29,305 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:35905 2024-11-25T12:57:29,308 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35905 connecting to ZooKeeper ensemble=127.0.0.1:60142 2024-11-25T12:57:29,309 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T12:57:29,312 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T12:57:29,328 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:359050x0, quorum=127.0.0.1:60142, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-25T12:57:29,329 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:359050x0, quorum=127.0.0.1:60142, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T12:57:29,331 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35905-0x101735baab60001 connected 2024-11-25T12:57:29,334 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-25T12:57:29,349 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-25T12:57:29,354 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35905-0x101735baab60001, quorum=127.0.0.1:60142, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-25T12:57:29,369 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35905-0x101735baab60001, quorum=127.0.0.1:60142, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-25T12:57:29,376 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35905 2024-11-25T12:57:29,376 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35905 2024-11-25T12:57:29,379 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35905 2024-11-25T12:57:29,383 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35905 2024-11-25T12:57:29,387 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35905 2024-11-25T12:57:29,409 DEBUG [M:0;1ff8009e463e:46289 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;1ff8009e463e:46289 2024-11-25T12:57:29,410 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/1ff8009e463e,46289,1732539448369 2024-11-25T12:57:29,428 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46289-0x101735baab60000, quorum=127.0.0.1:60142, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T12:57:29,429 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35905-0x101735baab60001, quorum=127.0.0.1:60142, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T12:57:29,432 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46289-0x101735baab60000, quorum=127.0.0.1:60142, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/1ff8009e463e,46289,1732539448369 2024-11-25T12:57:29,472 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35905-0x101735baab60001, quorum=127.0.0.1:60142, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-25T12:57:29,472 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46289-0x101735baab60000, quorum=127.0.0.1:60142, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:57:29,472 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35905-0x101735baab60001, quorum=127.0.0.1:60142, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:57:29,478 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46289-0x101735baab60000, quorum=127.0.0.1:60142, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-25T12:57:29,481 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/1ff8009e463e,46289,1732539448369 from backup master directory 2024-11-25T12:57:29,521 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46289-0x101735baab60000, quorum=127.0.0.1:60142, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/1ff8009e463e,46289,1732539448369 2024-11-25T12:57:29,521 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35905-0x101735baab60001, quorum=127.0.0.1:60142, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T12:57:29,521 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46289-0x101735baab60000, quorum=127.0.0.1:60142, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T12:57:29,522 WARN [master/1ff8009e463e:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-25T12:57:29,523 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=1ff8009e463e,46289,1732539448369 2024-11-25T12:57:29,531 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-25T12:57:29,539 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-25T12:57:29,616 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/hbase.id] with ID: cb972fd5-cdcb-422b-acfe-41fc80d301d9 2024-11-25T12:57:29,617 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/.tmp/hbase.id 2024-11-25T12:57:29,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741826_1002 (size=42) 2024-11-25T12:57:29,632 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/.tmp/hbase.id]:[hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/hbase.id] 2024-11-25T12:57:29,692 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T12:57:29,698 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-25T12:57:29,725 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 25ms. 2024-11-25T12:57:29,745 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35905-0x101735baab60001, quorum=127.0.0.1:60142, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:57:29,745 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46289-0x101735baab60000, quorum=127.0.0.1:60142, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:57:29,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741827_1003 (size=196) 2024-11-25T12:57:29,783 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T12:57:29,787 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-25T12:57:29,807 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:29,812 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T12:57:29,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741828_1004 (size=1189) 2024-11-25T12:57:29,912 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/MasterData/data/master/store 2024-11-25T12:57:29,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741829_1005 (size=34) 2024-11-25T12:57:30,345 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-25T12:57:30,349 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:57:30,353 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-25T12:57:30,353 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T12:57:30,353 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T12:57:30,358 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-25T12:57:30,358 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T12:57:30,358 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T12:57:30,360 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732539450352Disabling compacts and flushes for region at 1732539450353 (+1 ms)Disabling writes for close at 1732539450358 (+5 ms)Writing region close event to WAL at 1732539450358Closed at 1732539450358 2024-11-25T12:57:30,365 WARN [master/1ff8009e463e:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/MasterData/data/master/store/.initializing 2024-11-25T12:57:30,365 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/MasterData/WALs/1ff8009e463e,46289,1732539448369 2024-11-25T12:57:30,385 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T12:57:30,412 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1ff8009e463e%2C46289%2C1732539448369, suffix=, logDir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/MasterData/WALs/1ff8009e463e,46289,1732539448369, archiveDir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/MasterData/oldWALs, maxLogs=10 2024-11-25T12:57:30,451 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/MasterData/WALs/1ff8009e463e,46289,1732539448369/1ff8009e463e%2C46289%2C1732539448369.1732539450420, exclude list is [], retry=0 2024-11-25T12:57:30,484 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38683,DS-012750d6-b62b-46ef-b486-505acfd2e005,DISK] 2024-11-25T12:57:30,488 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-25T12:57:30,545 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/MasterData/WALs/1ff8009e463e,46289,1732539448369/1ff8009e463e%2C46289%2C1732539448369.1732539450420 2024-11-25T12:57:30,547 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37633:37633)] 2024-11-25T12:57:30,547 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-25T12:57:30,548 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:57:30,552 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T12:57:30,554 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T12:57:30,597 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T12:57:30,623 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-25T12:57:30,626 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:57:30,629 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T12:57:30,629 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T12:57:30,633 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-25T12:57:30,633 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:57:30,634 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:57:30,634 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T12:57:30,637 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-25T12:57:30,637 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:57:30,638 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:57:30,638 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T12:57:30,641 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-25T12:57:30,641 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:57:30,642 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:57:30,643 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T12:57:30,646 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-25T12:57:30,647 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-25T12:57:30,652 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T12:57:30,653 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T12:57:30,656 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-25T12:57:30,660 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T12:57:30,664 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T12:57:30,666 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61874535, jitterRate=-0.07799758017063141}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-25T12:57:30,673 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732539450570Initializing all the Stores at 1732539450572 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732539450573 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732539450573Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732539450574 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732539450574Cleaning up temporary data from old regions at 1732539450653 (+79 ms)Region opened successfully at 1732539450673 (+20 ms) 2024-11-25T12:57:30,675 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-25T12:57:30,705 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7328efbb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=1ff8009e463e/172.17.0.3:0 2024-11-25T12:57:30,733 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-25T12:57:30,742 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-25T12:57:30,742 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-25T12:57:30,745 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-25T12:57:30,746 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-25T12:57:30,751 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-25T12:57:30,751 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-25T12:57:30,776 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-25T12:57:30,784 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46289-0x101735baab60000, quorum=127.0.0.1:60142, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-25T12:57:30,819 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-25T12:57:30,822 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-25T12:57:30,823 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46289-0x101735baab60000, quorum=127.0.0.1:60142, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-25T12:57:30,827 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-25T12:57:30,830 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-25T12:57:30,833 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46289-0x101735baab60000, quorum=127.0.0.1:60142, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-25T12:57:30,844 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-25T12:57:30,845 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46289-0x101735baab60000, quorum=127.0.0.1:60142, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-25T12:57:30,853 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-25T12:57:30,875 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46289-0x101735baab60000, quorum=127.0.0.1:60142, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-25T12:57:30,886 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-25T12:57:30,902 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35905-0x101735baab60001, quorum=127.0.0.1:60142, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-25T12:57:30,902 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46289-0x101735baab60000, quorum=127.0.0.1:60142, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-25T12:57:30,902 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35905-0x101735baab60001, quorum=127.0.0.1:60142, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:57:30,902 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46289-0x101735baab60000, quorum=127.0.0.1:60142, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:57:30,904 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=1ff8009e463e,46289,1732539448369, sessionid=0x101735baab60000, setting cluster-up flag (Was=false) 2024-11-25T12:57:30,944 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35905-0x101735baab60001, quorum=127.0.0.1:60142, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:57:30,944 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46289-0x101735baab60000, quorum=127.0.0.1:60142, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:57:30,970 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-25T12:57:30,972 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=1ff8009e463e,46289,1732539448369 2024-11-25T12:57:30,986 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35905-0x101735baab60001, quorum=127.0.0.1:60142, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:57:30,986 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46289-0x101735baab60000, quorum=127.0.0.1:60142, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:57:31,019 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-25T12:57:31,021 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=1ff8009e463e,46289,1732539448369 2024-11-25T12:57:31,029 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-25T12:57:31,102 INFO [RS:0;1ff8009e463e:35905 {}] regionserver.HRegionServer(746): ClusterId : cb972fd5-cdcb-422b-acfe-41fc80d301d9 2024-11-25T12:57:31,105 DEBUG [RS:0;1ff8009e463e:35905 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-25T12:57:31,113 DEBUG [RS:0;1ff8009e463e:35905 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-25T12:57:31,113 DEBUG [RS:0;1ff8009e463e:35905 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-25T12:57:31,120 DEBUG [RS:0;1ff8009e463e:35905 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-25T12:57:31,121 DEBUG [RS:0;1ff8009e463e:35905 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@631f8675, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=1ff8009e463e/172.17.0.3:0 2024-11-25T12:57:31,125 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-25T12:57:31,134 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-25T12:57:31,135 DEBUG [RS:0;1ff8009e463e:35905 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;1ff8009e463e:35905 2024-11-25T12:57:31,138 INFO [RS:0;1ff8009e463e:35905 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-25T12:57:31,138 INFO [RS:0;1ff8009e463e:35905 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-25T12:57:31,138 DEBUG [RS:0;1ff8009e463e:35905 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-25T12:57:31,140 INFO [RS:0;1ff8009e463e:35905 {}] regionserver.HRegionServer(2659): reportForDuty to master=1ff8009e463e,46289,1732539448369 with port=35905, startcode=1732539449246 2024-11-25T12:57:31,141 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-25T12:57:31,147 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 1ff8009e463e,46289,1732539448369 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-25T12:57:31,151 DEBUG [RS:0;1ff8009e463e:35905 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-25T12:57:31,162 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/1ff8009e463e:0, corePoolSize=5, maxPoolSize=5 2024-11-25T12:57:31,162 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/1ff8009e463e:0, corePoolSize=5, maxPoolSize=5 2024-11-25T12:57:31,163 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/1ff8009e463e:0, corePoolSize=5, maxPoolSize=5 2024-11-25T12:57:31,163 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/1ff8009e463e:0, corePoolSize=5, maxPoolSize=5 2024-11-25T12:57:31,163 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/1ff8009e463e:0, corePoolSize=10, maxPoolSize=10 2024-11-25T12:57:31,163 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:57:31,163 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/1ff8009e463e:0, corePoolSize=2, maxPoolSize=2 2024-11-25T12:57:31,164 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:57:31,173 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-25T12:57:31,173 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-25T12:57:31,184 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:57:31,185 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-25T12:57:31,192 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732539481191 2024-11-25T12:57:31,193 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-25T12:57:31,194 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-25T12:57:31,198 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-25T12:57:31,198 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-25T12:57:31,198 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-25T12:57:31,199 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-25T12:57:31,203 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-25T12:57:31,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741831_1007 (size=1321) 2024-11-25T12:57:31,208 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-25T12:57:31,209 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-25T12:57:31,210 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-25T12:57:31,210 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-25T12:57:31,210 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a 2024-11-25T12:57:31,213 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40557, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-25T12:57:31,215 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-25T12:57:31,216 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-25T12:57:31,222 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46289 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:31,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741832_1008 (size=32) 2024-11-25T12:57:31,248 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:57:31,255 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/1ff8009e463e:0:becomeActiveMaster-HFileCleaner.large.0-1732539451229,5,FailOnTimeoutGroup] 2024-11-25T12:57:31,256 DEBUG [RS:0;1ff8009e463e:35905 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-25T12:57:31,256 WARN [RS:0;1ff8009e463e:35905 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-25T12:57:31,258 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/1ff8009e463e:0:becomeActiveMaster-HFileCleaner.small.0-1732539451256,5,FailOnTimeoutGroup] 2024-11-25T12:57:31,258 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-25T12:57:31,258 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-25T12:57:31,260 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-25T12:57:31,260 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-25T12:57:31,260 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-25T12:57:31,280 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-25T12:57:31,280 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:57:31,282 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T12:57:31,282 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-25T12:57:31,294 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-25T12:57:31,294 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:57:31,296 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T12:57:31,296 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-25T12:57:31,308 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-25T12:57:31,308 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:57:31,309 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T12:57:31,310 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-25T12:57:31,313 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-25T12:57:31,314 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:57:31,315 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T12:57:31,315 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-25T12:57:31,317 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/hbase/meta/1588230740 2024-11-25T12:57:31,318 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/hbase/meta/1588230740 2024-11-25T12:57:31,323 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-25T12:57:31,323 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-25T12:57:31,325 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-25T12:57:31,329 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-25T12:57:31,340 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T12:57:31,341 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67259429, jitterRate=0.0022435933351516724}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-25T12:57:31,346 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732539451248Initializing all the Stores at 1732539451252 (+4 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732539451252Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732539451259 (+7 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732539451259Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732539451259Cleaning up temporary data from old regions at 1732539451323 (+64 ms)Region opened successfully at 1732539451346 (+23 ms) 2024-11-25T12:57:31,346 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-25T12:57:31,347 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-25T12:57:31,347 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-25T12:57:31,347 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-25T12:57:31,347 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-25T12:57:31,349 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-25T12:57:31,349 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732539451346Disabling compacts and flushes for region at 1732539451346Disabling writes for close at 1732539451347 (+1 ms)Writing region close event to WAL at 1732539451348 (+1 ms)Closed at 1732539451349 (+1 ms) 2024-11-25T12:57:31,358 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-25T12:57:31,358 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-25T12:57:31,359 INFO [RS:0;1ff8009e463e:35905 {}] regionserver.HRegionServer(2659): reportForDuty to master=1ff8009e463e,46289,1732539448369 with port=35905, startcode=1732539449246 2024-11-25T12:57:31,365 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46289 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 1ff8009e463e,35905,1732539449246 2024-11-25T12:57:31,366 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-25T12:57:31,368 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46289 {}] master.ServerManager(517): Registering regionserver=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:31,379 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-25T12:57:31,388 DEBUG [RS:0;1ff8009e463e:35905 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a 2024-11-25T12:57:31,388 DEBUG [RS:0;1ff8009e463e:35905 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:32849 2024-11-25T12:57:31,388 DEBUG [RS:0;1ff8009e463e:35905 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-25T12:57:31,396 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-25T12:57:31,548 WARN [1ff8009e463e:46289 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T12:57:31,565 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46289-0x101735baab60000, quorum=127.0.0.1:60142, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-25T12:57:31,572 DEBUG [RS:0;1ff8009e463e:35905 {}] zookeeper.ZKUtil(111): regionserver:35905-0x101735baab60001, quorum=127.0.0.1:60142, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/1ff8009e463e,35905,1732539449246 2024-11-25T12:57:31,572 WARN [RS:0;1ff8009e463e:35905 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-25T12:57:31,572 INFO [RS:0;1ff8009e463e:35905 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T12:57:31,573 DEBUG [RS:0;1ff8009e463e:35905 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/WALs/1ff8009e463e,35905,1732539449246 2024-11-25T12:57:31,578 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [1ff8009e463e,35905,1732539449246] 2024-11-25T12:57:31,601 INFO [RS:0;1ff8009e463e:35905 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-25T12:57:31,617 INFO [RS:0;1ff8009e463e:35905 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-25T12:57:31,621 INFO [RS:0;1ff8009e463e:35905 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-25T12:57:31,621 INFO [RS:0;1ff8009e463e:35905 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T12:57:31,623 INFO [RS:0;1ff8009e463e:35905 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-25T12:57:31,628 INFO [RS:0;1ff8009e463e:35905 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-25T12:57:31,629 INFO [RS:0;1ff8009e463e:35905 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-25T12:57:31,630 DEBUG [RS:0;1ff8009e463e:35905 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:57:31,630 DEBUG [RS:0;1ff8009e463e:35905 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:57:31,630 DEBUG [RS:0;1ff8009e463e:35905 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:57:31,630 DEBUG [RS:0;1ff8009e463e:35905 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:57:31,631 DEBUG [RS:0;1ff8009e463e:35905 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:57:31,631 DEBUG [RS:0;1ff8009e463e:35905 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/1ff8009e463e:0, corePoolSize=2, maxPoolSize=2 2024-11-25T12:57:31,631 DEBUG [RS:0;1ff8009e463e:35905 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:57:31,631 DEBUG [RS:0;1ff8009e463e:35905 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:57:31,631 DEBUG [RS:0;1ff8009e463e:35905 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:57:31,631 DEBUG [RS:0;1ff8009e463e:35905 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:57:31,631 DEBUG [RS:0;1ff8009e463e:35905 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:57:31,632 DEBUG [RS:0;1ff8009e463e:35905 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:57:31,632 DEBUG [RS:0;1ff8009e463e:35905 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/1ff8009e463e:0, corePoolSize=3, maxPoolSize=3 2024-11-25T12:57:31,632 DEBUG [RS:0;1ff8009e463e:35905 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0, corePoolSize=3, maxPoolSize=3 2024-11-25T12:57:31,633 INFO [RS:0;1ff8009e463e:35905 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T12:57:31,633 INFO [RS:0;1ff8009e463e:35905 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T12:57:31,633 INFO [RS:0;1ff8009e463e:35905 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T12:57:31,633 INFO [RS:0;1ff8009e463e:35905 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-25T12:57:31,634 INFO [RS:0;1ff8009e463e:35905 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-25T12:57:31,634 INFO [RS:0;1ff8009e463e:35905 {}] hbase.ChoreService(168): Chore ScheduledChore name=1ff8009e463e,35905,1732539449246-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-25T12:57:31,653 INFO [RS:0;1ff8009e463e:35905 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-25T12:57:31,655 INFO [RS:0;1ff8009e463e:35905 {}] hbase.ChoreService(168): Chore ScheduledChore name=1ff8009e463e,35905,1732539449246-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T12:57:31,655 INFO [RS:0;1ff8009e463e:35905 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T12:57:31,656 INFO [RS:0;1ff8009e463e:35905 {}] regionserver.Replication(171): 1ff8009e463e,35905,1732539449246 started 2024-11-25T12:57:31,674 INFO [RS:0;1ff8009e463e:35905 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T12:57:31,674 INFO [RS:0;1ff8009e463e:35905 {}] regionserver.HRegionServer(1482): Serving as 1ff8009e463e,35905,1732539449246, RpcServer on 1ff8009e463e/172.17.0.3:35905, sessionid=0x101735baab60001 2024-11-25T12:57:31,675 DEBUG [RS:0;1ff8009e463e:35905 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-25T12:57:31,675 DEBUG [RS:0;1ff8009e463e:35905 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 1ff8009e463e,35905,1732539449246 2024-11-25T12:57:31,675 DEBUG [RS:0;1ff8009e463e:35905 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1ff8009e463e,35905,1732539449246' 2024-11-25T12:57:31,676 DEBUG [RS:0;1ff8009e463e:35905 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-25T12:57:31,677 DEBUG [RS:0;1ff8009e463e:35905 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-25T12:57:31,678 DEBUG [RS:0;1ff8009e463e:35905 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-25T12:57:31,678 DEBUG [RS:0;1ff8009e463e:35905 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-25T12:57:31,678 DEBUG [RS:0;1ff8009e463e:35905 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 1ff8009e463e,35905,1732539449246 2024-11-25T12:57:31,678 DEBUG [RS:0;1ff8009e463e:35905 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1ff8009e463e,35905,1732539449246' 2024-11-25T12:57:31,678 DEBUG [RS:0;1ff8009e463e:35905 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-25T12:57:31,679 DEBUG [RS:0;1ff8009e463e:35905 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-25T12:57:31,680 DEBUG [RS:0;1ff8009e463e:35905 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-25T12:57:31,680 INFO [RS:0;1ff8009e463e:35905 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-25T12:57:31,680 INFO [RS:0;1ff8009e463e:35905 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-25T12:57:31,784 INFO [RS:0;1ff8009e463e:35905 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T12:57:31,788 INFO [RS:0;1ff8009e463e:35905 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1ff8009e463e%2C35905%2C1732539449246, suffix=, logDir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/WALs/1ff8009e463e,35905,1732539449246, archiveDir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/oldWALs, maxLogs=32 2024-11-25T12:57:31,804 DEBUG [RS:0;1ff8009e463e:35905 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/WALs/1ff8009e463e,35905,1732539449246/1ff8009e463e%2C35905%2C1732539449246.1732539451791, exclude list is [], retry=0 2024-11-25T12:57:31,809 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38683,DS-012750d6-b62b-46ef-b486-505acfd2e005,DISK] 2024-11-25T12:57:31,812 INFO [RS:0;1ff8009e463e:35905 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/WALs/1ff8009e463e,35905,1732539449246/1ff8009e463e%2C35905%2C1732539449246.1732539451791 2024-11-25T12:57:31,813 DEBUG [RS:0;1ff8009e463e:35905 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37633:37633)] 2024-11-25T12:57:32,050 DEBUG [1ff8009e463e:46289 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-25T12:57:32,061 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:32,069 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1ff8009e463e,35905,1732539449246, state=OPENING 2024-11-25T12:57:32,086 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-25T12:57:32,094 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46289-0x101735baab60000, quorum=127.0.0.1:60142, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:57:32,094 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35905-0x101735baab60001, quorum=127.0.0.1:60142, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:57:32,096 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T12:57:32,096 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T12:57:32,097 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-25T12:57:32,099 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=1ff8009e463e,35905,1732539449246}] 2024-11-25T12:57:32,275 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-25T12:57:32,279 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41045, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-25T12:57:32,292 INFO [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-25T12:57:32,292 INFO [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T12:57:32,293 INFO [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-25T12:57:32,297 INFO [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1ff8009e463e%2C35905%2C1732539449246.meta, suffix=.meta, logDir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/WALs/1ff8009e463e,35905,1732539449246, archiveDir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/oldWALs, maxLogs=32 2024-11-25T12:57:32,319 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/WALs/1ff8009e463e,35905,1732539449246/1ff8009e463e%2C35905%2C1732539449246.meta.1732539452300.meta, exclude list is [], retry=0 2024-11-25T12:57:32,324 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38683,DS-012750d6-b62b-46ef-b486-505acfd2e005,DISK] 2024-11-25T12:57:32,332 INFO [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/WALs/1ff8009e463e,35905,1732539449246/1ff8009e463e%2C35905%2C1732539449246.meta.1732539452300.meta 2024-11-25T12:57:32,335 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37633:37633)] 2024-11-25T12:57:32,336 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-25T12:57:32,338 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-25T12:57:32,341 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-25T12:57:32,346 INFO [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-25T12:57:32,351 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-25T12:57:32,352 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:57:32,352 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-25T12:57:32,353 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-25T12:57:32,357 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-25T12:57:32,359 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-25T12:57:32,359 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:57:32,361 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T12:57:32,361 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-25T12:57:32,363 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-25T12:57:32,363 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:57:32,364 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T12:57:32,364 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-25T12:57:32,366 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-25T12:57:32,366 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:57:32,367 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T12:57:32,367 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-25T12:57:32,369 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-25T12:57:32,369 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:57:32,370 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T12:57:32,371 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-25T12:57:32,373 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/hbase/meta/1588230740 2024-11-25T12:57:32,377 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/hbase/meta/1588230740 2024-11-25T12:57:32,380 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-25T12:57:32,380 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-25T12:57:32,381 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-25T12:57:32,384 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-25T12:57:32,386 INFO [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65182743, jitterRate=-0.02870143949985504}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-25T12:57:32,387 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-25T12:57:32,389 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732539452353Writing region info on filesystem at 1732539452353Initializing all the Stores at 1732539452356 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732539452356Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732539452357 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732539452357Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732539452357Cleaning up temporary data from old regions at 1732539452380 (+23 ms)Running coprocessor post-open hooks at 1732539452387 (+7 ms)Region opened successfully at 1732539452389 (+2 ms) 2024-11-25T12:57:32,398 INFO [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732539452266 2024-11-25T12:57:32,412 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-25T12:57:32,413 INFO [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-25T12:57:32,415 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:32,418 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1ff8009e463e,35905,1732539449246, state=OPEN 2024-11-25T12:57:32,454 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35905-0x101735baab60001, quorum=127.0.0.1:60142, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T12:57:32,454 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T12:57:32,455 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46289-0x101735baab60000, quorum=127.0.0.1:60142, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T12:57:32,455 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T12:57:32,456 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:32,466 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-25T12:57:32,467 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1ff8009e463e,35905,1732539449246 in 357 msec 2024-11-25T12:57:32,478 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-25T12:57:32,478 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 1.1010 sec 2024-11-25T12:57:32,480 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-25T12:57:32,480 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-25T12:57:32,510 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:57:32,512 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,35905,1732539449246, seqNum=-1] 2024-11-25T12:57:32,542 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:57:32,547 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57727, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:57:32,589 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.5100 sec 2024-11-25T12:57:32,590 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732539452589, completionTime=-1 2024-11-25T12:57:32,593 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-25T12:57:32,593 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-25T12:57:32,633 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-25T12:57:32,633 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732539512633 2024-11-25T12:57:32,633 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732539572633 2024-11-25T12:57:32,633 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 40 msec 2024-11-25T12:57:32,636 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1ff8009e463e,46289,1732539448369-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T12:57:32,636 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1ff8009e463e,46289,1732539448369-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T12:57:32,636 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1ff8009e463e,46289,1732539448369-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T12:57:32,638 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-1ff8009e463e:46289, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T12:57:32,638 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-25T12:57:32,639 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-25T12:57:32,646 DEBUG [master/1ff8009e463e:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-25T12:57:32,671 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.148sec 2024-11-25T12:57:32,672 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-25T12:57:32,673 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-25T12:57:32,674 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-25T12:57:32,675 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-25T12:57:32,675 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-25T12:57:32,676 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1ff8009e463e,46289,1732539448369-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-25T12:57:32,677 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1ff8009e463e,46289,1732539448369-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-25T12:57:32,685 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-25T12:57:32,686 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-25T12:57:32,687 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1ff8009e463e,46289,1732539448369-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T12:57:32,716 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@132591a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:57:32,718 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-25T12:57:32,718 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-25T12:57:32,721 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ff8009e463e,46289,-1 for getting cluster id 2024-11-25T12:57:32,724 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:57:32,731 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb972fd5-cdcb-422b-acfe-41fc80d301d9' 2024-11-25T12:57:32,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:57:32,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb972fd5-cdcb-422b-acfe-41fc80d301d9" 2024-11-25T12:57:32,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5314b86, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:57:32,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ff8009e463e,46289,-1] 2024-11-25T12:57:32,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:57:32,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:57:32,741 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43112, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:57:32,744 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39c236f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:57:32,745 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:57:32,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,35905,1732539449246, seqNum=-1] 2024-11-25T12:57:32,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:57:32,756 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40604, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:57:32,773 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=1ff8009e463e,46289,1732539448369 2024-11-25T12:57:32,789 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=701, ProcessCount=11, AvailableMemoryMB=2036 2024-11-25T12:57:32,821 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-25T12:57:32,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 1ff8009e463e,46289,1732539448369 2024-11-25T12:57:32,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2e3afd56 2024-11-25T12:57:32,834 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-25T12:57:32,838 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43126, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-25T12:57:32,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-25T12:57:32,846 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T12:57:32,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-25T12:57:32,865 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T12:57:32,868 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:57:32,872 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T12:57:32,879 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-25T12:57:32,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T12:57:32,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741835_1011 (size=959) 2024-11-25T12:57:33,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T12:57:33,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T12:57:33,308 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a 2024-11-25T12:57:33,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741836_1012 (size=53) 2024-11-25T12:57:33,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T12:57:33,753 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:57:33,754 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing e7fe900c90851bb165c5d4e631eedb85, disabling compactions & flushes 2024-11-25T12:57:33,754 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:33,754 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:33,754 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. after waiting 0 ms 2024-11-25T12:57:33,754 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:33,754 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:33,754 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for e7fe900c90851bb165c5d4e631eedb85: Waiting for close lock at 1732539453754Disabling compacts and flushes for region at 1732539453754Disabling writes for close at 1732539453754Writing region close event to WAL at 1732539453754Closed at 1732539453754 2024-11-25T12:57:33,758 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T12:57:33,764 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732539453758"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732539453758"}]},"ts":"1732539453758"} 2024-11-25T12:57:33,771 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T12:57:33,777 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T12:57:33,781 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732539453777"}]},"ts":"1732539453777"} 2024-11-25T12:57:33,791 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-25T12:57:33,793 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e7fe900c90851bb165c5d4e631eedb85, ASSIGN}] 2024-11-25T12:57:33,796 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e7fe900c90851bb165c5d4e631eedb85, ASSIGN 2024-11-25T12:57:33,799 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e7fe900c90851bb165c5d4e631eedb85, ASSIGN; state=OFFLINE, location=1ff8009e463e,35905,1732539449246; forceNewPlan=false, retain=false 2024-11-25T12:57:33,952 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=e7fe900c90851bb165c5d4e631eedb85, regionState=OPENING, regionLocation=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:33,971 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e7fe900c90851bb165c5d4e631eedb85, ASSIGN because future has completed 2024-11-25T12:57:33,972 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246}] 2024-11-25T12:57:34,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T12:57:34,140 INFO [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:34,141 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} 2024-11-25T12:57:34,142 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e7fe900c90851bb165c5d4e631eedb85 2024-11-25T12:57:34,142 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:57:34,142 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for e7fe900c90851bb165c5d4e631eedb85 2024-11-25T12:57:34,142 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for e7fe900c90851bb165c5d4e631eedb85 2024-11-25T12:57:34,147 INFO [StoreOpener-e7fe900c90851bb165c5d4e631eedb85-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e7fe900c90851bb165c5d4e631eedb85 2024-11-25T12:57:34,155 INFO [StoreOpener-e7fe900c90851bb165c5d4e631eedb85-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e7fe900c90851bb165c5d4e631eedb85 columnFamilyName A 2024-11-25T12:57:34,155 DEBUG [StoreOpener-e7fe900c90851bb165c5d4e631eedb85-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:57:34,159 INFO [StoreOpener-e7fe900c90851bb165c5d4e631eedb85-1 {}] regionserver.HStore(327): Store=e7fe900c90851bb165c5d4e631eedb85/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:57:34,159 INFO [StoreOpener-e7fe900c90851bb165c5d4e631eedb85-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e7fe900c90851bb165c5d4e631eedb85 2024-11-25T12:57:34,170 INFO [StoreOpener-e7fe900c90851bb165c5d4e631eedb85-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e7fe900c90851bb165c5d4e631eedb85 columnFamilyName B 2024-11-25T12:57:34,170 DEBUG [StoreOpener-e7fe900c90851bb165c5d4e631eedb85-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:57:34,171 INFO [StoreOpener-e7fe900c90851bb165c5d4e631eedb85-1 {}] regionserver.HStore(327): Store=e7fe900c90851bb165c5d4e631eedb85/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:57:34,172 INFO [StoreOpener-e7fe900c90851bb165c5d4e631eedb85-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e7fe900c90851bb165c5d4e631eedb85 2024-11-25T12:57:34,175 INFO [StoreOpener-e7fe900c90851bb165c5d4e631eedb85-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e7fe900c90851bb165c5d4e631eedb85 columnFamilyName C 2024-11-25T12:57:34,175 DEBUG [StoreOpener-e7fe900c90851bb165c5d4e631eedb85-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:57:34,176 INFO [StoreOpener-e7fe900c90851bb165c5d4e631eedb85-1 {}] regionserver.HStore(327): Store=e7fe900c90851bb165c5d4e631eedb85/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:57:34,176 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for e7fe900c90851bb165c5d4e631eedb85 2024-11-25T12:57:34,178 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85 2024-11-25T12:57:34,179 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85 2024-11-25T12:57:34,186 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for e7fe900c90851bb165c5d4e631eedb85 2024-11-25T12:57:34,187 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for e7fe900c90851bb165c5d4e631eedb85 2024-11-25T12:57:34,192 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-25T12:57:34,195 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for e7fe900c90851bb165c5d4e631eedb85 2024-11-25T12:57:34,199 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T12:57:34,200 INFO [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened e7fe900c90851bb165c5d4e631eedb85; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67372390, jitterRate=0.0039268434047698975}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-25T12:57:34,201 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e7fe900c90851bb165c5d4e631eedb85 2024-11-25T12:57:34,202 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for e7fe900c90851bb165c5d4e631eedb85: Running coprocessor pre-open hook at 1732539454143Writing region info on filesystem at 1732539454143Initializing all the Stores at 1732539454145 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732539454145Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732539454145Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732539454145Cleaning up temporary data from old regions at 1732539454187 (+42 ms)Running coprocessor post-open hooks at 1732539454201 (+14 ms)Region opened successfully at 1732539454202 (+1 ms) 2024-11-25T12:57:34,205 INFO [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., pid=6, masterSystemTime=1732539454132 2024-11-25T12:57:34,210 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:34,210 INFO [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:34,211 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=e7fe900c90851bb165c5d4e631eedb85, regionState=OPEN, openSeqNum=2, regionLocation=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:34,217 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 because future has completed 2024-11-25T12:57:34,225 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-25T12:57:34,225 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 in 248 msec 2024-11-25T12:57:34,234 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-25T12:57:34,234 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e7fe900c90851bb165c5d4e631eedb85, ASSIGN in 432 msec 2024-11-25T12:57:34,236 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T12:57:34,236 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732539454236"}]},"ts":"1732539454236"} 2024-11-25T12:57:34,240 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-25T12:57:34,242 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T12:57:34,247 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.3920 sec 2024-11-25T12:57:35,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T12:57:35,052 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:57:35,068 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4938db69, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:57:35,068 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ff8009e463e,46289,-1 for getting cluster id 2024-11-25T12:57:35,068 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:57:35,082 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb972fd5-cdcb-422b-acfe-41fc80d301d9' 2024-11-25T12:57:35,085 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:57:35,085 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb972fd5-cdcb-422b-acfe-41fc80d301d9" 2024-11-25T12:57:35,085 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54ede97e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:57:35,085 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ff8009e463e,46289,-1] 2024-11-25T12:57:35,086 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:57:35,087 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:57:35,089 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33392, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:57:35,092 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cc70ab1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:57:35,093 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:57:35,096 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,35905,1732539449246, seqNum=-1] 2024-11-25T12:57:35,098 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:57:35,103 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54946, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:57:35,106 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-25T12:57:35,111 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 1ff8009e463e,46289,1732539448369 2024-11-25T12:57:35,111 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1820fead 2024-11-25T12:57:35,114 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-25T12:57:35,118 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33400, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-25T12:57:35,136 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f0e98fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:57:35,136 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ff8009e463e,46289,-1 for getting cluster id 2024-11-25T12:57:35,136 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:57:35,145 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb972fd5-cdcb-422b-acfe-41fc80d301d9' 2024-11-25T12:57:35,145 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:57:35,145 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb972fd5-cdcb-422b-acfe-41fc80d301d9" 2024-11-25T12:57:35,146 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c7def95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:57:35,146 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ff8009e463e,46289,-1] 2024-11-25T12:57:35,146 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:57:35,147 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:57:35,149 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33416, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:57:35,150 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c373c50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:57:35,151 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69066494, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:57:35,152 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ff8009e463e,46289,-1 for getting cluster id 2024-11-25T12:57:35,152 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:57:35,154 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb972fd5-cdcb-422b-acfe-41fc80d301d9' 2024-11-25T12:57:35,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:57:35,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb972fd5-cdcb-422b-acfe-41fc80d301d9" 2024-11-25T12:57:35,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@551748ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:57:35,157 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ff8009e463e,46289,-1] 2024-11-25T12:57:35,157 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:57:35,167 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:57:35,171 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33432, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:57:35,174 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4834e559, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:57:35,176 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f00f692, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:57:35,176 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ff8009e463e,46289,-1 for getting cluster id 2024-11-25T12:57:35,176 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:57:35,191 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb972fd5-cdcb-422b-acfe-41fc80d301d9' 2024-11-25T12:57:35,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:57:35,193 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb972fd5-cdcb-422b-acfe-41fc80d301d9" 2024-11-25T12:57:35,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75ca9e14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:57:35,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ff8009e463e,46289,-1] 2024-11-25T12:57:35,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:57:35,203 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:57:35,209 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33450, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:57:35,211 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13d05b08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:57:35,215 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@388c382, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:57:35,215 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ff8009e463e,46289,-1 for getting cluster id 2024-11-25T12:57:35,216 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:57:35,230 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb972fd5-cdcb-422b-acfe-41fc80d301d9' 2024-11-25T12:57:35,231 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:57:35,231 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb972fd5-cdcb-422b-acfe-41fc80d301d9" 2024-11-25T12:57:35,231 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35afed03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:57:35,231 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ff8009e463e,46289,-1] 2024-11-25T12:57:35,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:57:35,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:57:35,238 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33466, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:57:35,240 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@108777b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:57:35,244 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b48425, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:57:35,244 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ff8009e463e,46289,-1 for getting cluster id 2024-11-25T12:57:35,244 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:57:35,246 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb972fd5-cdcb-422b-acfe-41fc80d301d9' 2024-11-25T12:57:35,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:57:35,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb972fd5-cdcb-422b-acfe-41fc80d301d9" 2024-11-25T12:57:35,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68aa74df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:57:35,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ff8009e463e,46289,-1] 2024-11-25T12:57:35,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:57:35,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:57:35,251 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33480, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:57:35,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9e0aade, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:57:35,280 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@761fbe54, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:57:35,280 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ff8009e463e,46289,-1 for getting cluster id 2024-11-25T12:57:35,280 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:57:35,282 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb972fd5-cdcb-422b-acfe-41fc80d301d9' 2024-11-25T12:57:35,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:57:35,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb972fd5-cdcb-422b-acfe-41fc80d301d9" 2024-11-25T12:57:35,284 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f39f01e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:57:35,284 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ff8009e463e,46289,-1] 2024-11-25T12:57:35,284 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:57:35,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:57:35,289 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33502, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:57:35,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ddc1f05, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:57:35,293 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b9a31d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:57:35,293 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ff8009e463e,46289,-1 for getting cluster id 2024-11-25T12:57:35,293 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:57:35,294 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb972fd5-cdcb-422b-acfe-41fc80d301d9' 2024-11-25T12:57:35,295 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:57:35,295 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb972fd5-cdcb-422b-acfe-41fc80d301d9" 2024-11-25T12:57:35,295 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66b367dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:57:35,295 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ff8009e463e,46289,-1] 2024-11-25T12:57:35,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:57:35,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:57:35,298 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33518, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:57:35,299 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@327794d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:57:35,310 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36df70c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:57:35,310 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ff8009e463e,46289,-1 for getting cluster id 2024-11-25T12:57:35,311 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:57:35,312 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb972fd5-cdcb-422b-acfe-41fc80d301d9' 2024-11-25T12:57:35,314 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:57:35,315 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb972fd5-cdcb-422b-acfe-41fc80d301d9" 2024-11-25T12:57:35,315 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@555e9a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:57:35,315 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ff8009e463e,46289,-1] 2024-11-25T12:57:35,315 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:57:35,316 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:57:35,317 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33536, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:57:35,318 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1058791b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:57:35,323 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e8cc673, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:57:35,324 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ff8009e463e,46289,-1 for getting cluster id 2024-11-25T12:57:35,324 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:57:35,325 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb972fd5-cdcb-422b-acfe-41fc80d301d9' 2024-11-25T12:57:35,325 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:57:35,326 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb972fd5-cdcb-422b-acfe-41fc80d301d9" 2024-11-25T12:57:35,326 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19b6d532, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:57:35,326 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ff8009e463e,46289,-1] 2024-11-25T12:57:35,326 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:57:35,327 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:57:35,329 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33554, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:57:35,330 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@394100ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:57:35,334 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:57:35,336 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:57:35,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,35905,1732539449246, seqNum=-1] 2024-11-25T12:57:35,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:57:35,338 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,35905,1732539449246, seqNum=-1] 2024-11-25T12:57:35,338 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:57:35,339 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:57:35,340 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54954, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:57:35,340 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54970, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:57:35,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,35905,1732539449246, seqNum=-1] 2024-11-25T12:57:35,341 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:57:35,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:57:35,343 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,35905,1732539449246, seqNum=-1] 2024-11-25T12:57:35,343 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54984, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:57:35,343 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:57:35,345 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54986, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:57:35,347 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2] 2024-11-25T12:57:35,348 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2] 2024-11-25T12:57:35,350 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2] 2024-11-25T12:57:35,351 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2] 2024-11-25T12:57:35,353 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:57:35,356 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,35905,1732539449246, seqNum=-1] 2024-11-25T12:57:35,358 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:57:35,360 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54990, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:57:35,360 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:57:35,365 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,35905,1732539449246, seqNum=-1] 2024-11-25T12:57:35,366 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:57:35,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2] 2024-11-25T12:57:35,372 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54994, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:57:35,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2] 2024-11-25T12:57:35,378 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:57:35,380 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,35905,1732539449246, seqNum=-1] 2024-11-25T12:57:35,381 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:57:35,383 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55004, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:57:35,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2] 2024-11-25T12:57:35,390 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:57:35,397 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,35905,1732539449246, seqNum=-1] 2024-11-25T12:57:35,397 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:57:35,399 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55018, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:57:35,401 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:57:35,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2] 2024-11-25T12:57:35,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-25T12:57:35,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-25T12:57:35,409 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:57:35,410 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:57:35,410 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,35905,1732539449246, seqNum=-1] 2024-11-25T12:57:35,411 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:57:35,413 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55028, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:57:35,415 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:57:35,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2] 2024-11-25T12:57:35,417 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:57:35,468 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:57:35,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on e7fe900c90851bb165c5d4e631eedb85 2024-11-25T12:57:35,473 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e7fe900c90851bb165c5d4e631eedb85 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-25T12:57:35,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-25T12:57:35,598 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-25T12:57:35,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:35,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. as already flushing 2024-11-25T12:57:35,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:35,621 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:35,622 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/a65a3e9e016b48769a3e1064776c6c4b is 50, key is test_row_0/A:col10/1732539455459/Put/seqid=0 2024-11-25T12:57:35,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:35,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:35,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741837_1013 (size=19021) 2024-11-25T12:57:35,683 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/a65a3e9e016b48769a3e1064776c6c4b 2024-11-25T12:57:35,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:35,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:35,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54990 deadline: 1732539515682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:35,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:35,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55028 deadline: 1732539515678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:35,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:35,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:35,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:35,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:35,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:35,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:35,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54984 deadline: 1732539515685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:35,704 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:35,704 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:35,704 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:35,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:35,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55004 deadline: 1732539515698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:35,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:35,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:35,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:35,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:35,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54986 deadline: 1732539515701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:35,735 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:35,735 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:35,735 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:35,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-25T12:57:35,799 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/575539a855ad4ac0a7f1840d0476f6a1 is 50, key is test_row_0/B:col10/1732539455459/Put/seqid=0 2024-11-25T12:57:35,800 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-25T12:57:35,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:35,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. as already flushing 2024-11-25T12:57:35,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:35,807 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:35,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:35,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:35,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:35,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:35,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55028 deadline: 1732539515819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:35,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54990 deadline: 1732539515819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:35,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:35,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54984 deadline: 1732539515820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:35,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:35,836 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:35,836 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:35,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:35,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55004 deadline: 1732539515831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:35,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:35,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:35,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:35,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:35,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:35,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:35,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:35,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:35,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:35,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:35,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54986 deadline: 1732539515842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:35,853 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:35,853 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:35,853 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:35,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741838_1014 (size=12001) 2024-11-25T12:57:35,871 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/575539a855ad4ac0a7f1840d0476f6a1 2024-11-25T12:57:35,924 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/77d7dfae1582430b957c9c92c665a62c is 50, key is test_row_0/C:col10/1732539455459/Put/seqid=0 2024-11-25T12:57:35,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741839_1015 (size=12001) 2024-11-25T12:57:35,982 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-25T12:57:35,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:35,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. as already flushing 2024-11-25T12:57:35,983 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/77d7dfae1582430b957c9c92c665a62c 2024-11-25T12:57:35,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:35,984 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:35,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:35,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:36,007 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/a65a3e9e016b48769a3e1064776c6c4b as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/a65a3e9e016b48769a3e1064776c6c4b 2024-11-25T12:57:36,023 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/a65a3e9e016b48769a3e1064776c6c4b, entries=300, sequenceid=13, filesize=18.6 K 2024-11-25T12:57:36,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/575539a855ad4ac0a7f1840d0476f6a1 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/575539a855ad4ac0a7f1840d0476f6a1 2024-11-25T12:57:36,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-25T12:57:36,052 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/575539a855ad4ac0a7f1840d0476f6a1, entries=150, sequenceid=13, filesize=11.7 K 2024-11-25T12:57:36,055 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/77d7dfae1582430b957c9c92c665a62c as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/77d7dfae1582430b957c9c92c665a62c 2024-11-25T12:57:36,081 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/77d7dfae1582430b957c9c92c665a62c, entries=150, sequenceid=13, filesize=11.7 K 2024-11-25T12:57:36,097 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for e7fe900c90851bb165c5d4e631eedb85 in 623ms, sequenceid=13, compaction requested=false 2024-11-25T12:57:36,098 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-25T12:57:36,107 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e7fe900c90851bb165c5d4e631eedb85: 2024-11-25T12:57:36,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on e7fe900c90851bb165c5d4e631eedb85 2024-11-25T12:57:36,128 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:57:36,129 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e7fe900c90851bb165c5d4e631eedb85 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-25T12:57:36,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-25T12:57:36,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:36,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. as already flushing 2024-11-25T12:57:36,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:36,144 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:36,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:36,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:36,161 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/e91ac446bd494d89b700c4fd131a5168 is 50, key is test_row_0/A:col10/1732539455683/Put/seqid=0 2024-11-25T12:57:36,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:36,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55004 deadline: 1732539516151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:36,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:36,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54986 deadline: 1732539516153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:36,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:36,191 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:36,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:36,192 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:36,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:36,192 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:36,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:36,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55028 deadline: 1732539516167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:36,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:36,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:36,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:36,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:36,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54990 deadline: 1732539516171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:36,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:36,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54984 deadline: 1732539516182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:36,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:36,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:36,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:36,200 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:36,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:36,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:36,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741840_1016 (size=16681) 2024-11-25T12:57:36,227 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/e91ac446bd494d89b700c4fd131a5168 2024-11-25T12:57:36,269 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/09625318e20042b0a9f46bbc51858d8f is 50, key is test_row_0/B:col10/1732539455683/Put/seqid=0 2024-11-25T12:57:36,305 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-25T12:57:36,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:36,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. as already flushing 2024-11-25T12:57:36,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:36,306 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:36,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:36,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:36,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741841_1017 (size=12001) 2024-11-25T12:57:36,315 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/09625318e20042b0a9f46bbc51858d8f 2024-11-25T12:57:36,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:36,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55004 deadline: 1732539516301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:36,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:36,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:36,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:36,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:36,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54986 deadline: 1732539516301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:36,320 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:36,320 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:36,320 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:36,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:36,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54984 deadline: 1732539516312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:36,330 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:36,330 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:36,330 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:36,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:36,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54990 deadline: 1732539516312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:36,332 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:36,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:36,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:36,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:36,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55028 deadline: 1732539516312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:36,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:36,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:36,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:36,348 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/cf30b6d70cef4015991561c23700f8b7 is 50, key is test_row_0/C:col10/1732539455683/Put/seqid=0 2024-11-25T12:57:36,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741842_1018 (size=12001) 2024-11-25T12:57:36,432 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/cf30b6d70cef4015991561c23700f8b7 2024-11-25T12:57:36,452 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/e91ac446bd494d89b700c4fd131a5168 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/e91ac446bd494d89b700c4fd131a5168 2024-11-25T12:57:36,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-25T12:57:36,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:36,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. as already flushing 2024-11-25T12:57:36,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:36,463 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:36,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:36,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:36,470 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/e91ac446bd494d89b700c4fd131a5168, entries=250, sequenceid=40, filesize=16.3 K 2024-11-25T12:57:36,472 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/09625318e20042b0a9f46bbc51858d8f as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/09625318e20042b0a9f46bbc51858d8f 2024-11-25T12:57:36,489 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/09625318e20042b0a9f46bbc51858d8f, entries=150, sequenceid=40, filesize=11.7 K 2024-11-25T12:57:36,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/cf30b6d70cef4015991561c23700f8b7 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/cf30b6d70cef4015991561c23700f8b7 2024-11-25T12:57:36,513 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/cf30b6d70cef4015991561c23700f8b7, entries=150, sequenceid=40, filesize=11.7 K 2024-11-25T12:57:36,517 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for e7fe900c90851bb165c5d4e631eedb85 in 388ms, sequenceid=40, compaction requested=false 2024-11-25T12:57:36,517 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e7fe900c90851bb165c5d4e631eedb85: 2024-11-25T12:57:36,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on e7fe900c90851bb165c5d4e631eedb85 2024-11-25T12:57:36,547 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:57:36,548 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e7fe900c90851bb165c5d4e631eedb85 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-25T12:57:36,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-25T12:57:36,569 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/af958271896a47769f71a2ec514c650e is 50, key is test_row_0/A:col10/1732539456543/Put/seqid=0 2024-11-25T12:57:36,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741843_1019 (size=14341) 2024-11-25T12:57:36,619 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-25T12:57:36,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:36,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. as already flushing 2024-11-25T12:57:36,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:36,620 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/af958271896a47769f71a2ec514c650e 2024-11-25T12:57:36,620 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:36,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:36,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:36,656 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/fbf6d8c94a2f46e5bb1acc1bcae65984 is 50, key is test_row_0/B:col10/1732539456543/Put/seqid=0 2024-11-25T12:57:36,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741844_1020 (size=12001) 2024-11-25T12:57:36,710 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/fbf6d8c94a2f46e5bb1acc1bcae65984 2024-11-25T12:57:36,765 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/fc7c6b3e700043ef8fac8f8c72e63e9e is 50, key is test_row_0/C:col10/1732539456543/Put/seqid=0 2024-11-25T12:57:36,784 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-25T12:57:36,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:36,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. as already flushing 2024-11-25T12:57:36,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:36,785 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:36,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:36,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:36,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741845_1021 (size=12001) 2024-11-25T12:57:36,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:36,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54990 deadline: 1732539516861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:36,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:36,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:36,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:36,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:36,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54986 deadline: 1732539516922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:36,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:36,951 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:36,951 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:36,951 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:36,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54984 deadline: 1732539516928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:36,954 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:36,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:36,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:36,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:36,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55004 deadline: 1732539516944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:36,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:36,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:36,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:36,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:36,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55028 deadline: 1732539516945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:36,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:36,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:36,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:36,980 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-25T12:57:36,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:36,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. as already flushing 2024-11-25T12:57:36,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:36,981 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:36,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:36,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:37,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:37,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54986 deadline: 1732539517062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:37,096 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:37,096 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:37,096 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:37,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:37,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54984 deadline: 1732539517063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:37,100 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:37,100 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:37,100 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:37,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:37,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54990 deadline: 1732539517063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:37,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:37,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:37,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:37,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:37,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55004 deadline: 1732539517083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:37,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:37,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:37,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:37,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:37,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55028 deadline: 1732539517083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:37,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:37,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:37,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:37,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-25T12:57:37,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:37,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. as already flushing 2024-11-25T12:57:37,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:37,137 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:37,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:37,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:37,249 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/fc7c6b3e700043ef8fac8f8c72e63e9e 2024-11-25T12:57:37,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/af958271896a47769f71a2ec514c650e as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/af958271896a47769f71a2ec514c650e 2024-11-25T12:57:37,292 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/af958271896a47769f71a2ec514c650e, entries=200, sequenceid=51, filesize=14.0 K 2024-11-25T12:57:37,293 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-25T12:57:37,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:37,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. as already flushing 2024-11-25T12:57:37,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:37,293 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:37,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:37,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:37,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/fbf6d8c94a2f46e5bb1acc1bcae65984 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/fbf6d8c94a2f46e5bb1acc1bcae65984 2024-11-25T12:57:37,313 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/fbf6d8c94a2f46e5bb1acc1bcae65984, entries=150, sequenceid=51, filesize=11.7 K 2024-11-25T12:57:37,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:37,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54986 deadline: 1732539517301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:37,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/fc7c6b3e700043ef8fac8f8c72e63e9e as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/fc7c6b3e700043ef8fac8f8c72e63e9e 2024-11-25T12:57:37,328 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:37,328 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:37,328 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:37,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:37,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54984 deadline: 1732539517311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:37,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:37,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55004 deadline: 1732539517312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:37,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:37,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:37,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:37,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:37,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:37,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54990 deadline: 1732539517311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:37,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:37,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:37,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:37,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:37,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:37,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:37,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55028 deadline: 1732539517328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:37,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:37,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:37,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:37,343 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/fc7c6b3e700043ef8fac8f8c72e63e9e, entries=150, sequenceid=51, filesize=11.7 K 2024-11-25T12:57:37,346 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for e7fe900c90851bb165c5d4e631eedb85 in 798ms, sequenceid=51, compaction requested=true 2024-11-25T12:57:37,346 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e7fe900c90851bb165c5d4e631eedb85: 2024-11-25T12:57:37,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7fe900c90851bb165c5d4e631eedb85:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:57:37,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:57:37,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7fe900c90851bb165c5d4e631eedb85:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:57:37,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:57:37,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7fe900c90851bb165c5d4e631eedb85:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:57:37,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:57:37,363 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:57:37,364 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:57:37,367 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:57:37,368 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): e7fe900c90851bb165c5d4e631eedb85/B is initiating minor compaction (all files) 2024-11-25T12:57:37,369 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7fe900c90851bb165c5d4e631eedb85/B in TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:37,370 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/575539a855ad4ac0a7f1840d0476f6a1, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/09625318e20042b0a9f46bbc51858d8f, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/fbf6d8c94a2f46e5bb1acc1bcae65984] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp, totalSize=35.2 K 2024-11-25T12:57:37,373 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 575539a855ad4ac0a7f1840d0476f6a1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732539455359 2024-11-25T12:57:37,374 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 09625318e20042b0a9f46bbc51858d8f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732539455683 2024-11-25T12:57:37,375 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting fbf6d8c94a2f46e5bb1acc1bcae65984, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732539456160 2024-11-25T12:57:37,379 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 50043 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:57:37,379 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1541): e7fe900c90851bb165c5d4e631eedb85/A is initiating minor compaction (all files) 2024-11-25T12:57:37,379 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7fe900c90851bb165c5d4e631eedb85/A in TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:37,381 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/a65a3e9e016b48769a3e1064776c6c4b, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/e91ac446bd494d89b700c4fd131a5168, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/af958271896a47769f71a2ec514c650e] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp, totalSize=48.9 K 2024-11-25T12:57:37,383 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting a65a3e9e016b48769a3e1064776c6c4b, keycount=300, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732539455359 2024-11-25T12:57:37,385 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting e91ac446bd494d89b700c4fd131a5168, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732539455667 2024-11-25T12:57:37,386 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting af958271896a47769f71a2ec514c650e, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732539456160 2024-11-25T12:57:37,441 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7fe900c90851bb165c5d4e631eedb85#B#compaction#9 average throughput is 0.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:57:37,442 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/00b5380c8cf841e2a19380578a94339d is 50, key is test_row_0/B:col10/1732539456543/Put/seqid=0 2024-11-25T12:57:37,450 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-25T12:57:37,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:37,450 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing e7fe900c90851bb165c5d4e631eedb85 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-25T12:57:37,453 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7fe900c90851bb165c5d4e631eedb85#A#compaction#10 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:57:37,454 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/949279ac08824960b0c6139042677c3a is 50, key is test_row_0/A:col10/1732539456543/Put/seqid=0 2024-11-25T12:57:37,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741847_1023 (size=12104) 2024-11-25T12:57:37,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741846_1022 (size=12104) 2024-11-25T12:57:37,542 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/949279ac08824960b0c6139042677c3a as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/949279ac08824960b0c6139042677c3a 2024-11-25T12:57:37,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/0271da18e24741c8a5198ded16b64256 is 50, key is test_row_0/A:col10/1732539456933/Put/seqid=0 2024-11-25T12:57:37,561 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/00b5380c8cf841e2a19380578a94339d as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/00b5380c8cf841e2a19380578a94339d 2024-11-25T12:57:37,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-25T12:57:37,591 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7fe900c90851bb165c5d4e631eedb85/B of e7fe900c90851bb165c5d4e631eedb85 into 00b5380c8cf841e2a19380578a94339d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:57:37,591 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7fe900c90851bb165c5d4e631eedb85: 2024-11-25T12:57:37,592 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7fe900c90851bb165c5d4e631eedb85/A of e7fe900c90851bb165c5d4e631eedb85 into 949279ac08824960b0c6139042677c3a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:57:37,592 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7fe900c90851bb165c5d4e631eedb85: 2024-11-25T12:57:37,596 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., storeName=e7fe900c90851bb165c5d4e631eedb85/B, priority=13, startTime=1732539457361; duration=0sec 2024-11-25T12:57:37,597 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:57:37,597 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7fe900c90851bb165c5d4e631eedb85:B 2024-11-25T12:57:37,598 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:57:37,598 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., storeName=e7fe900c90851bb165c5d4e631eedb85/A, priority=13, startTime=1732539457347; duration=0sec 2024-11-25T12:57:37,599 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:57:37,599 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7fe900c90851bb165c5d4e631eedb85:A 2024-11-25T12:57:37,605 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:57:37,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741848_1024 (size=12001) 2024-11-25T12:57:37,607 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): e7fe900c90851bb165c5d4e631eedb85/C is initiating minor compaction (all files) 2024-11-25T12:57:37,607 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7fe900c90851bb165c5d4e631eedb85/C in TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:37,607 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/0271da18e24741c8a5198ded16b64256 2024-11-25T12:57:37,608 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/77d7dfae1582430b957c9c92c665a62c, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/cf30b6d70cef4015991561c23700f8b7, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/fc7c6b3e700043ef8fac8f8c72e63e9e] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp, totalSize=35.2 K 2024-11-25T12:57:37,611 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 77d7dfae1582430b957c9c92c665a62c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732539455359 2024-11-25T12:57:37,612 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting cf30b6d70cef4015991561c23700f8b7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732539455683 2024-11-25T12:57:37,614 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting fc7c6b3e700043ef8fac8f8c72e63e9e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732539456160 2024-11-25T12:57:37,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/7458dfc0991a4c908e5e6bcc7ef9f353 is 50, key is test_row_0/B:col10/1732539456933/Put/seqid=0 2024-11-25T12:57:37,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. as already flushing 2024-11-25T12:57:37,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on e7fe900c90851bb165c5d4e631eedb85 2024-11-25T12:57:37,686 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7fe900c90851bb165c5d4e631eedb85#C#compaction#13 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:57:37,687 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/b04f42b3a5f9431d908a0311dbf03b07 is 50, key is test_row_0/C:col10/1732539456543/Put/seqid=0 2024-11-25T12:57:37,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741849_1025 (size=12001) 2024-11-25T12:57:37,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:37,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54986 deadline: 1732539517693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:37,711 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:37,711 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:37,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:37,711 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:37,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55004 deadline: 1732539517693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:37,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:37,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:37,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:37,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:37,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55028 deadline: 1732539517702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:37,723 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:37,723 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:37,723 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:37,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:37,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54990 deadline: 1732539517708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:37,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:37,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:37,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:37,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54984 deadline: 1732539517710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:37,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:37,725 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:37,725 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:37,725 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:37,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741850_1026 (size=12104) 2024-11-25T12:57:37,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:37,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54986 deadline: 1732539517821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:37,828 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:37,828 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:37,828 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:37,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:37,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55004 deadline: 1732539517821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:37,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:37,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:37,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:37,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:37,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54990 deadline: 1732539517834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:37,838 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:37,839 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:37,839 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:37,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:37,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55028 deadline: 1732539517839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:37,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:37,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:37,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:37,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:37,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54984 deadline: 1732539517840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:37,846 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:37,846 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:37,846 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:37,865 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T12:57:38,039 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-25T12:57:38,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:38,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55004 deadline: 1732539518041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:38,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:38,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:38,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:38,046 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:38,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54986 deadline: 1732539518041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:38,047 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:38,048 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:38,048 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:38,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:38,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55028 deadline: 1732539518051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:38,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:38,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:38,066 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:38,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:38,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54984 deadline: 1732539518051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:38,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:38,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54990 deadline: 1732539518051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:38,069 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:38,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:38,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:38,087 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:38,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:38,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:38,109 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/7458dfc0991a4c908e5e6bcc7ef9f353 2024-11-25T12:57:38,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/c6367c2ee92a475ea4751cc9ae7efc01 is 50, key is test_row_0/C:col10/1732539456933/Put/seqid=0 2024-11-25T12:57:38,165 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/b04f42b3a5f9431d908a0311dbf03b07 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/b04f42b3a5f9431d908a0311dbf03b07 2024-11-25T12:57:38,181 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7fe900c90851bb165c5d4e631eedb85/C of e7fe900c90851bb165c5d4e631eedb85 into b04f42b3a5f9431d908a0311dbf03b07(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:57:38,181 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7fe900c90851bb165c5d4e631eedb85: 2024-11-25T12:57:38,181 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., storeName=e7fe900c90851bb165c5d4e631eedb85/C, priority=13, startTime=1732539457361; duration=0sec 2024-11-25T12:57:38,181 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:57:38,181 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7fe900c90851bb165c5d4e631eedb85:C 2024-11-25T12:57:38,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741851_1027 (size=12001) 2024-11-25T12:57:38,216 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/c6367c2ee92a475ea4751cc9ae7efc01 2024-11-25T12:57:38,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/0271da18e24741c8a5198ded16b64256 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/0271da18e24741c8a5198ded16b64256 2024-11-25T12:57:38,274 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/0271da18e24741c8a5198ded16b64256, entries=150, sequenceid=76, filesize=11.7 K 2024-11-25T12:57:38,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/7458dfc0991a4c908e5e6bcc7ef9f353 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/7458dfc0991a4c908e5e6bcc7ef9f353 2024-11-25T12:57:38,292 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/7458dfc0991a4c908e5e6bcc7ef9f353, entries=150, sequenceid=76, filesize=11.7 K 2024-11-25T12:57:38,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/c6367c2ee92a475ea4751cc9ae7efc01 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/c6367c2ee92a475ea4751cc9ae7efc01 2024-11-25T12:57:38,310 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/c6367c2ee92a475ea4751cc9ae7efc01, entries=150, sequenceid=76, filesize=11.7 K 2024-11-25T12:57:38,313 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for e7fe900c90851bb165c5d4e631eedb85 in 862ms, sequenceid=76, compaction requested=false 2024-11-25T12:57:38,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for e7fe900c90851bb165c5d4e631eedb85: 2024-11-25T12:57:38,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:38,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-25T12:57:38,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-25T12:57:38,323 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-25T12:57:38,324 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8990 sec 2024-11-25T12:57:38,334 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.9270 sec 2024-11-25T12:57:38,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on e7fe900c90851bb165c5d4e631eedb85 2024-11-25T12:57:38,356 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:57:38,356 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e7fe900c90851bb165c5d4e631eedb85 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-25T12:57:38,375 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/8acc07bcaeaf4fbcbbc9371ec17844c9 is 50, key is test_row_0/A:col10/1732539458353/Put/seqid=0 2024-11-25T12:57:38,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741852_1028 (size=16681) 2024-11-25T12:57:38,416 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/8acc07bcaeaf4fbcbbc9371ec17844c9 2024-11-25T12:57:38,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:38,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54986 deadline: 1732539518418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:38,435 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:38,435 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:38,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:38,435 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:38,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55028 deadline: 1732539518421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:38,436 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:38,436 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:38,436 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:38,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:38,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55004 deadline: 1732539518429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:38,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:38,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:38,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:38,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:38,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54990 deadline: 1732539518433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:38,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:38,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:38,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:38,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:38,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54984 deadline: 1732539518434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:38,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:38,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:38,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:38,449 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/363763aca5cc4c48b5348ba8837f2b1a is 50, key is test_row_0/B:col10/1732539458353/Put/seqid=0 2024-11-25T12:57:38,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741853_1029 (size=12001) 2024-11-25T12:57:38,503 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/363763aca5cc4c48b5348ba8837f2b1a 2024-11-25T12:57:38,535 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/3fec7ba60a634b419778d9ae2c79cb51 is 50, key is test_row_0/C:col10/1732539458353/Put/seqid=0 2024-11-25T12:57:38,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:38,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54986 deadline: 1732539518540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:38,546 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:38,546 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:38,546 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:38,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:38,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55028 deadline: 1732539518540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:38,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:38,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:38,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:38,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:38,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54990 deadline: 1732539518551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:38,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:38,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:38,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:38,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:38,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:38,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55004 deadline: 1732539518551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:38,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54984 deadline: 1732539518551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:38,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:38,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:38,559 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:38,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:38,560 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:38,560 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:38,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741854_1030 (size=12001) 2024-11-25T12:57:38,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:38,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54986 deadline: 1732539518750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:38,760 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:38,760 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:38,760 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:38,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:38,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55028 deadline: 1732539518761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:38,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:38,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:38,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:38,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:38,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54990 deadline: 1732539518770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:38,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:38,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:38,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:38,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:38,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54984 deadline: 1732539518770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:38,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:38,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55004 deadline: 1732539518770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:38,780 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:38,780 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:38,781 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:38,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:38,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:38,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:38,805 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-25T12:57:38,805 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-25T12:57:38,807 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-25T12:57:38,807 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-25T12:57:38,808 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-25T12:57:38,808 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-25T12:57:38,809 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-25T12:57:38,809 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-25T12:57:38,999 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/3fec7ba60a634b419778d9ae2c79cb51 2024-11-25T12:57:39,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/8acc07bcaeaf4fbcbbc9371ec17844c9 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/8acc07bcaeaf4fbcbbc9371ec17844c9 2024-11-25T12:57:39,042 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/8acc07bcaeaf4fbcbbc9371ec17844c9, entries=250, sequenceid=92, filesize=16.3 K 2024-11-25T12:57:39,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/363763aca5cc4c48b5348ba8837f2b1a as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/363763aca5cc4c48b5348ba8837f2b1a 2024-11-25T12:57:39,062 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/363763aca5cc4c48b5348ba8837f2b1a, entries=150, sequenceid=92, filesize=11.7 K 2024-11-25T12:57:39,067 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/3fec7ba60a634b419778d9ae2c79cb51 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/3fec7ba60a634b419778d9ae2c79cb51 2024-11-25T12:57:39,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:39,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54986 deadline: 1732539519070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:39,085 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:39,085 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:39,085 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:39,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:39,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55028 deadline: 1732539519071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:39,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:39,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:39,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:39,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:39,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54984 deadline: 1732539519091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:39,100 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:39,100 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:39,100 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:39,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:39,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54990 deadline: 1732539519094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:39,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:39,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:39,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:39,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:39,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55004 deadline: 1732539519091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:39,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:39,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:39,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:39,114 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/3fec7ba60a634b419778d9ae2c79cb51, entries=150, sequenceid=92, filesize=11.7 K 2024-11-25T12:57:39,118 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for e7fe900c90851bb165c5d4e631eedb85 in 762ms, sequenceid=92, compaction requested=true 2024-11-25T12:57:39,119 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e7fe900c90851bb165c5d4e631eedb85: 2024-11-25T12:57:39,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7fe900c90851bb165c5d4e631eedb85:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:57:39,119 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:57:39,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:57:39,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7fe900c90851bb165c5d4e631eedb85:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:57:39,119 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:57:39,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:57:39,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7fe900c90851bb165c5d4e631eedb85:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:57:39,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:57:39,122 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:57:39,122 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): e7fe900c90851bb165c5d4e631eedb85/B is initiating minor compaction (all files) 2024-11-25T12:57:39,122 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7fe900c90851bb165c5d4e631eedb85/B in TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:39,123 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/00b5380c8cf841e2a19380578a94339d, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/7458dfc0991a4c908e5e6bcc7ef9f353, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/363763aca5cc4c48b5348ba8837f2b1a] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp, totalSize=35.3 K 2024-11-25T12:57:39,124 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40786 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:57:39,124 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1541): e7fe900c90851bb165c5d4e631eedb85/A is initiating minor compaction (all files) 2024-11-25T12:57:39,124 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7fe900c90851bb165c5d4e631eedb85/A in TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:39,125 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/949279ac08824960b0c6139042677c3a, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/0271da18e24741c8a5198ded16b64256, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/8acc07bcaeaf4fbcbbc9371ec17844c9] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp, totalSize=39.8 K 2024-11-25T12:57:39,131 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 949279ac08824960b0c6139042677c3a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732539456160 2024-11-25T12:57:39,143 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 00b5380c8cf841e2a19380578a94339d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732539456160 2024-11-25T12:57:39,144 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0271da18e24741c8a5198ded16b64256, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732539456852 2024-11-25T12:57:39,145 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8acc07bcaeaf4fbcbbc9371ec17844c9, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732539457672 2024-11-25T12:57:39,145 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 7458dfc0991a4c908e5e6bcc7ef9f353, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732539456852 2024-11-25T12:57:39,148 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 363763aca5cc4c48b5348ba8837f2b1a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732539457703 2024-11-25T12:57:39,171 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7fe900c90851bb165c5d4e631eedb85#A#compaction#18 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:57:39,172 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/637eb22cbc2945888e7d4069c6c5f794 is 50, key is test_row_0/A:col10/1732539458353/Put/seqid=0 2024-11-25T12:57:39,180 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7fe900c90851bb165c5d4e631eedb85#B#compaction#19 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:57:39,181 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/0f12ca889b7342f2bd8855de5203c166 is 50, key is test_row_0/B:col10/1732539458353/Put/seqid=0 2024-11-25T12:57:39,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741855_1031 (size=12207) 2024-11-25T12:57:39,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741856_1032 (size=12207) 2024-11-25T12:57:39,278 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/637eb22cbc2945888e7d4069c6c5f794 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/637eb22cbc2945888e7d4069c6c5f794 2024-11-25T12:57:39,296 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/0f12ca889b7342f2bd8855de5203c166 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/0f12ca889b7342f2bd8855de5203c166 2024-11-25T12:57:39,300 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7fe900c90851bb165c5d4e631eedb85/A of e7fe900c90851bb165c5d4e631eedb85 into 637eb22cbc2945888e7d4069c6c5f794(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:57:39,301 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7fe900c90851bb165c5d4e631eedb85: 2024-11-25T12:57:39,301 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., storeName=e7fe900c90851bb165c5d4e631eedb85/A, priority=13, startTime=1732539459119; duration=0sec 2024-11-25T12:57:39,301 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:57:39,301 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7fe900c90851bb165c5d4e631eedb85:A 2024-11-25T12:57:39,301 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:57:39,307 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:57:39,307 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1541): e7fe900c90851bb165c5d4e631eedb85/C is initiating minor compaction (all files) 2024-11-25T12:57:39,308 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7fe900c90851bb165c5d4e631eedb85/C in TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:39,308 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/b04f42b3a5f9431d908a0311dbf03b07, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/c6367c2ee92a475ea4751cc9ae7efc01, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/3fec7ba60a634b419778d9ae2c79cb51] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp, totalSize=35.3 K 2024-11-25T12:57:39,309 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting b04f42b3a5f9431d908a0311dbf03b07, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732539456160 2024-11-25T12:57:39,313 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7fe900c90851bb165c5d4e631eedb85/B of e7fe900c90851bb165c5d4e631eedb85 into 0f12ca889b7342f2bd8855de5203c166(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:57:39,314 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7fe900c90851bb165c5d4e631eedb85: 2024-11-25T12:57:39,314 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., storeName=e7fe900c90851bb165c5d4e631eedb85/B, priority=13, startTime=1732539459119; duration=0sec 2024-11-25T12:57:39,314 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:57:39,314 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7fe900c90851bb165c5d4e631eedb85:B 2024-11-25T12:57:39,314 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6367c2ee92a475ea4751cc9ae7efc01, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732539456852 2024-11-25T12:57:39,315 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3fec7ba60a634b419778d9ae2c79cb51, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732539457703 2024-11-25T12:57:39,362 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7fe900c90851bb165c5d4e631eedb85#C#compaction#20 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:57:39,363 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/873bb666be514c04ad8b935de5bbbccd is 50, key is test_row_0/C:col10/1732539458353/Put/seqid=0 2024-11-25T12:57:39,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741857_1033 (size=12207) 2024-11-25T12:57:39,393 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/873bb666be514c04ad8b935de5bbbccd as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/873bb666be514c04ad8b935de5bbbccd 2024-11-25T12:57:39,407 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7fe900c90851bb165c5d4e631eedb85/C of e7fe900c90851bb165c5d4e631eedb85 into 873bb666be514c04ad8b935de5bbbccd(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:57:39,407 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7fe900c90851bb165c5d4e631eedb85: 2024-11-25T12:57:39,407 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., storeName=e7fe900c90851bb165c5d4e631eedb85/C, priority=13, startTime=1732539459119; duration=0sec 2024-11-25T12:57:39,408 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:57:39,408 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7fe900c90851bb165c5d4e631eedb85:C 2024-11-25T12:57:39,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-25T12:57:39,591 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:57:39,592 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:57:39,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-25T12:57:39,603 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:57:39,605 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:57:39,606 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:57:39,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-25T12:57:39,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-25T12:57:39,763 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-25T12:57:39,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:39,764 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing e7fe900c90851bb165c5d4e631eedb85 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-25T12:57:39,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on e7fe900c90851bb165c5d4e631eedb85 2024-11-25T12:57:39,768 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. as already flushing 2024-11-25T12:57:39,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:39,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55028 deadline: 1732539519790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:39,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:39,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:39,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:39,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54986 deadline: 1732539519792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:39,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:39,804 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:39,804 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:39,804 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:39,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/b2ac59dfdce744929a265975a24bd9fb is 50, key is test_row_0/A:col10/1732539458410/Put/seqid=0 2024-11-25T12:57:39,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:39,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54990 deadline: 1732539519801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:39,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:39,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:39,810 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:39,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:39,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55004 deadline: 1732539519803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:39,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:39,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:39,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:39,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:39,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54984 deadline: 1732539519803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:39,812 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:39,812 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:39,812 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:39,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741858_1034 (size=12001) 2024-11-25T12:57:39,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:39,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55028 deadline: 1732539519910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:39,918 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:39,918 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:39,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:39,918 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:39,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54986 deadline: 1732539519910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:39,919 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:39,919 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:39,919 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:39,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:39,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54990 deadline: 1732539519920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:39,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:39,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:39,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54984 deadline: 1732539519922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:39,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:39,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:39,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:39,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:39,928 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:39,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:39,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55004 deadline: 1732539519922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:39,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:39,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:39,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:39,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-25T12:57:40,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:40,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55028 deadline: 1732539520131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:40,137 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:40,137 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:40,137 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:40,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:40,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:40,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54986 deadline: 1732539520131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:40,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54990 deadline: 1732539520131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:40,139 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:40,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:40,139 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:40,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:40,139 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:40,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:40,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:40,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54984 deadline: 1732539520131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:40,140 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:40,140 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:40,140 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:40,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:40,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55004 deadline: 1732539520140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:40,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:40,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:40,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:40,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-25T12:57:40,261 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/b2ac59dfdce744929a265975a24bd9fb 2024-11-25T12:57:40,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/8744df911fba423397c05675909c341f is 50, key is test_row_0/B:col10/1732539458410/Put/seqid=0 2024-11-25T12:57:40,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741859_1035 (size=12001) 2024-11-25T12:57:40,361 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/8744df911fba423397c05675909c341f 2024-11-25T12:57:40,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/eb167a54f33d4a0e80ffa73416a1509d is 50, key is test_row_0/C:col10/1732539458410/Put/seqid=0 2024-11-25T12:57:40,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741860_1036 (size=12001) 2024-11-25T12:57:40,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:40,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55028 deadline: 1732539520444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:40,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:40,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:40,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:40,447 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/eb167a54f33d4a0e80ffa73416a1509d 2024-11-25T12:57:40,457 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:40,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:40,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54990 deadline: 1732539520451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:40,460 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:40,460 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:40,460 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:40,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:40,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54984 deadline: 1732539520455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:40,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:40,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:40,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:40,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54986 deadline: 1732539520452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:40,469 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:40,469 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:40,469 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:40,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:40,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55004 deadline: 1732539520460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:40,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:40,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:40,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:40,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/b2ac59dfdce744929a265975a24bd9fb as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/b2ac59dfdce744929a265975a24bd9fb 2024-11-25T12:57:40,504 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/b2ac59dfdce744929a265975a24bd9fb, entries=150, sequenceid=121, filesize=11.7 K 2024-11-25T12:57:40,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/8744df911fba423397c05675909c341f as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/8744df911fba423397c05675909c341f 2024-11-25T12:57:40,525 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/8744df911fba423397c05675909c341f, entries=150, sequenceid=121, filesize=11.7 K 2024-11-25T12:57:40,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/eb167a54f33d4a0e80ffa73416a1509d as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/eb167a54f33d4a0e80ffa73416a1509d 2024-11-25T12:57:40,544 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/eb167a54f33d4a0e80ffa73416a1509d, entries=150, sequenceid=121, filesize=11.7 K 2024-11-25T12:57:40,551 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for e7fe900c90851bb165c5d4e631eedb85 in 787ms, sequenceid=121, compaction requested=false 2024-11-25T12:57:40,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for e7fe900c90851bb165c5d4e631eedb85: 2024-11-25T12:57:40,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:40,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-25T12:57:40,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-25T12:57:40,558 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-25T12:57:40,558 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 950 msec 2024-11-25T12:57:40,567 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 968 msec 2024-11-25T12:57:40,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-25T12:57:40,761 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:57:40,762 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:57:40,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-25T12:57:40,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-25T12:57:40,766 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:57:40,768 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:57:40,768 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:57:40,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-25T12:57:40,923 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-25T12:57:40,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:40,924 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing e7fe900c90851bb165c5d4e631eedb85 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-25T12:57:40,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/f8b1967e7ead47f4a3293498476e527f is 50, key is test_row_0/A:col10/1732539459800/Put/seqid=0 2024-11-25T12:57:40,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. as already flushing 2024-11-25T12:57:40,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on e7fe900c90851bb165c5d4e631eedb85 2024-11-25T12:57:40,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741861_1037 (size=12101) 2024-11-25T12:57:41,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-25T12:57:41,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:41,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54984 deadline: 1732539521107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:41,132 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:41,132 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:41,132 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:41,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:41,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54986 deadline: 1732539521124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:41,145 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:41,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:41,145 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:41,145 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:41,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55004 deadline: 1732539521125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:41,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:41,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:41,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:41,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:41,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55028 deadline: 1732539521126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:41,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:41,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:41,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:41,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:41,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54990 deadline: 1732539521136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:41,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:41,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:41,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:41,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:41,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54984 deadline: 1732539521242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:41,256 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:41,256 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:41,256 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:41,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:41,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55004 deadline: 1732539521252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:41,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:41,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:41,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:41,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:41,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54986 deadline: 1732539521252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:41,259 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:41,259 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:41,259 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:41,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:41,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54990 deadline: 1732539521271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:41,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:41,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:41,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:41,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55028 deadline: 1732539521260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:41,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:41,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:41,284 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:41,284 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:41,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-25T12:57:41,395 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/f8b1967e7ead47f4a3293498476e527f 2024-11-25T12:57:41,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/2d0b71414d26400a965aed4d7d54205f is 50, key is test_row_0/B:col10/1732539459800/Put/seqid=0 2024-11-25T12:57:41,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741862_1038 (size=12101) 2024-11-25T12:57:41,450 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/2d0b71414d26400a965aed4d7d54205f 2024-11-25T12:57:41,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:41,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54984 deadline: 1732539521463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:41,470 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:41,470 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:41,470 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:41,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:41,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:41,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55004 deadline: 1732539521470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:41,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54986 deadline: 1732539521471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:41,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:41,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:41,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:41,484 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:41,484 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:41,484 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:41,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/48d34ab99b3741eb941bb6d28abeb522 is 50, key is test_row_0/C:col10/1732539459800/Put/seqid=0 2024-11-25T12:57:41,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:41,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54990 deadline: 1732539521490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:41,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:41,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:41,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:41,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:41,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55028 deadline: 1732539521490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:41,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:41,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:41,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:41,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741863_1039 (size=12101) 2024-11-25T12:57:41,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:41,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54984 deadline: 1732539521781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:41,795 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:41,795 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:41,795 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:41,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:41,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55004 deadline: 1732539521790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:41,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:41,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:41,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:41,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:41,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54986 deadline: 1732539521790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:41,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:41,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:41,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:41,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:41,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54990 deadline: 1732539521803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:41,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:41,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55028 deadline: 1732539521803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:41,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:41,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:41,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:41,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:41,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:41,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:41,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-25T12:57:41,927 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/48d34ab99b3741eb941bb6d28abeb522 2024-11-25T12:57:42,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/f8b1967e7ead47f4a3293498476e527f as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/f8b1967e7ead47f4a3293498476e527f 2024-11-25T12:57:42,041 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/f8b1967e7ead47f4a3293498476e527f, entries=150, sequenceid=132, filesize=11.8 K 2024-11-25T12:57:42,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/2d0b71414d26400a965aed4d7d54205f as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/2d0b71414d26400a965aed4d7d54205f 2024-11-25T12:57:42,079 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/2d0b71414d26400a965aed4d7d54205f, entries=150, sequenceid=132, filesize=11.8 K 2024-11-25T12:57:42,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/48d34ab99b3741eb941bb6d28abeb522 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/48d34ab99b3741eb941bb6d28abeb522 2024-11-25T12:57:42,091 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/48d34ab99b3741eb941bb6d28abeb522, entries=150, sequenceid=132, filesize=11.8 K 2024-11-25T12:57:42,093 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for e7fe900c90851bb165c5d4e631eedb85 in 1169ms, sequenceid=132, compaction requested=true 2024-11-25T12:57:42,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for e7fe900c90851bb165c5d4e631eedb85: 2024-11-25T12:57:42,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:42,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-25T12:57:42,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-25T12:57:42,101 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-25T12:57:42,102 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3290 sec 2024-11-25T12:57:42,113 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.3420 sec 2024-11-25T12:57:42,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on e7fe900c90851bb165c5d4e631eedb85 2024-11-25T12:57:42,303 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:57:42,303 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e7fe900c90851bb165c5d4e631eedb85 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-25T12:57:42,322 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/8176aa47487a4eaea3ceb972aa3dce44 is 50, key is test_row_0/A:col10/1732539462301/Put/seqid=0 2024-11-25T12:57:42,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741864_1040 (size=14541) 2024-11-25T12:57:42,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:42,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54990 deadline: 1732539522347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:42,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:42,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:42,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:42,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:42,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54984 deadline: 1732539522353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:42,372 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:42,372 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:42,372 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:42,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:42,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54986 deadline: 1732539522359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:42,375 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:42,375 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:42,375 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:42,375 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/8176aa47487a4eaea3ceb972aa3dce44 2024-11-25T12:57:42,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:42,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55028 deadline: 1732539522367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:42,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:42,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:42,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:42,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:42,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55004 deadline: 1732539522371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:42,382 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:42,382 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:42,382 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:42,436 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/fe50a6680a48498cbe4c91f033a0dc3f is 50, key is test_row_0/B:col10/1732539462301/Put/seqid=0 2024-11-25T12:57:42,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741865_1041 (size=12151) 2024-11-25T12:57:42,483 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/fe50a6680a48498cbe4c91f033a0dc3f 2024-11-25T12:57:42,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:42,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54986 deadline: 1732539522484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:42,489 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:42,489 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:42,489 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:42,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:42,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54984 deadline: 1732539522487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:42,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:42,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:42,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:42,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:42,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55028 deadline: 1732539522491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:42,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:42,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:42,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:42,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:42,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55004 deadline: 1732539522491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:42,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:42,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:42,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:42,525 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/690288bed1114f7c90f112241cf546e8 is 50, key is test_row_0/C:col10/1732539462301/Put/seqid=0 2024-11-25T12:57:42,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741866_1042 (size=12151) 2024-11-25T12:57:42,581 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/690288bed1114f7c90f112241cf546e8 2024-11-25T12:57:42,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/8176aa47487a4eaea3ceb972aa3dce44 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/8176aa47487a4eaea3ceb972aa3dce44 2024-11-25T12:57:42,698 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/8176aa47487a4eaea3ceb972aa3dce44, entries=200, sequenceid=159, filesize=14.2 K 2024-11-25T12:57:42,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/fe50a6680a48498cbe4c91f033a0dc3f as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/fe50a6680a48498cbe4c91f033a0dc3f 2024-11-25T12:57:42,721 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:42,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54986 deadline: 1732539522701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:42,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:42,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54984 deadline: 1732539522702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:42,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:42,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:42,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:42,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:42,725 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:42,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55028 deadline: 1732539522710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:42,725 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:42,725 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:42,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:42,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:42,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:42,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:42,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55004 deadline: 1732539522710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:42,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:42,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:42,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:42,731 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/fe50a6680a48498cbe4c91f033a0dc3f, entries=150, sequenceid=159, filesize=11.9 K 2024-11-25T12:57:42,748 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/690288bed1114f7c90f112241cf546e8 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/690288bed1114f7c90f112241cf546e8 2024-11-25T12:57:42,819 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/690288bed1114f7c90f112241cf546e8, entries=150, sequenceid=159, filesize=11.9 K 2024-11-25T12:57:42,822 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=46.96 KB/48090 for e7fe900c90851bb165c5d4e631eedb85 in 519ms, sequenceid=159, compaction requested=true 2024-11-25T12:57:42,822 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e7fe900c90851bb165c5d4e631eedb85: 2024-11-25T12:57:42,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7fe900c90851bb165c5d4e631eedb85:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:57:42,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:57:42,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7fe900c90851bb165c5d4e631eedb85:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:57:42,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-25T12:57:42,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7fe900c90851bb165c5d4e631eedb85:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:57:42,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-25T12:57:42,823 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:57:42,824 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:57:42,827 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48460 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:57:42,827 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): e7fe900c90851bb165c5d4e631eedb85/C is initiating minor compaction (all files) 2024-11-25T12:57:42,827 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7fe900c90851bb165c5d4e631eedb85/C in TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:42,827 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/873bb666be514c04ad8b935de5bbbccd, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/eb167a54f33d4a0e80ffa73416a1509d, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/48d34ab99b3741eb941bb6d28abeb522, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/690288bed1114f7c90f112241cf546e8] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp, totalSize=47.3 K 2024-11-25T12:57:42,828 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50850 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:57:42,828 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1541): e7fe900c90851bb165c5d4e631eedb85/A is initiating minor compaction (all files) 2024-11-25T12:57:42,828 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7fe900c90851bb165c5d4e631eedb85/A in TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:42,828 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/637eb22cbc2945888e7d4069c6c5f794, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/b2ac59dfdce744929a265975a24bd9fb, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/f8b1967e7ead47f4a3293498476e527f, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/8176aa47487a4eaea3ceb972aa3dce44] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp, totalSize=49.7 K 2024-11-25T12:57:42,829 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 873bb666be514c04ad8b935de5bbbccd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732539457703 2024-11-25T12:57:42,830 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 637eb22cbc2945888e7d4069c6c5f794, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732539457703 2024-11-25T12:57:42,831 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting eb167a54f33d4a0e80ffa73416a1509d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1732539458410 2024-11-25T12:57:42,831 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2ac59dfdce744929a265975a24bd9fb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1732539458410 2024-11-25T12:57:42,832 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 48d34ab99b3741eb941bb6d28abeb522, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732539459786 2024-11-25T12:57:42,832 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 690288bed1114f7c90f112241cf546e8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732539461112 2024-11-25T12:57:42,832 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8b1967e7ead47f4a3293498476e527f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732539459786 2024-11-25T12:57:42,833 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8176aa47487a4eaea3ceb972aa3dce44, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732539461106 2024-11-25T12:57:42,901 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7fe900c90851bb165c5d4e631eedb85#A#compaction#30 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:57:42,901 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/880aa15661fd4635988f299f7742698f is 50, key is test_row_0/A:col10/1732539462301/Put/seqid=0 2024-11-25T12:57:42,916 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7fe900c90851bb165c5d4e631eedb85#C#compaction#31 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:57:42,917 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/26ccdd28c269477886a50316129ed767 is 50, key is test_row_0/C:col10/1732539462301/Put/seqid=0 2024-11-25T12:57:42,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-25T12:57:42,921 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:57:42,922 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:57:42,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-25T12:57:42,927 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:57:42,929 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:57:42,929 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:57:42,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-25T12:57:42,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741867_1043 (size=12493) 2024-11-25T12:57:42,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741868_1044 (size=12493) 2024-11-25T12:57:43,005 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/26ccdd28c269477886a50316129ed767 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/26ccdd28c269477886a50316129ed767 2024-11-25T12:57:43,024 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e7fe900c90851bb165c5d4e631eedb85/C of e7fe900c90851bb165c5d4e631eedb85 into 26ccdd28c269477886a50316129ed767(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:57:43,024 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7fe900c90851bb165c5d4e631eedb85: 2024-11-25T12:57:43,024 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., storeName=e7fe900c90851bb165c5d4e631eedb85/C, priority=12, startTime=1732539462823; duration=0sec 2024-11-25T12:57:43,024 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:57:43,024 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7fe900c90851bb165c5d4e631eedb85:C 2024-11-25T12:57:43,024 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:57:43,027 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48460 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:57:43,027 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): e7fe900c90851bb165c5d4e631eedb85/B is initiating minor compaction (all files) 2024-11-25T12:57:43,027 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7fe900c90851bb165c5d4e631eedb85/B in TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:43,028 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/0f12ca889b7342f2bd8855de5203c166, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/8744df911fba423397c05675909c341f, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/2d0b71414d26400a965aed4d7d54205f, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/fe50a6680a48498cbe4c91f033a0dc3f] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp, totalSize=47.3 K 2024-11-25T12:57:43,029 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f12ca889b7342f2bd8855de5203c166, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732539457703 2024-11-25T12:57:43,029 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 8744df911fba423397c05675909c341f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1732539458410 2024-11-25T12:57:43,031 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d0b71414d26400a965aed4d7d54205f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732539459786 2024-11-25T12:57:43,032 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting fe50a6680a48498cbe4c91f033a0dc3f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732539461112 2024-11-25T12:57:43,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-25T12:57:43,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on e7fe900c90851bb165c5d4e631eedb85 2024-11-25T12:57:43,052 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:57:43,052 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e7fe900c90851bb165c5d4e631eedb85 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-25T12:57:43,063 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7fe900c90851bb165c5d4e631eedb85#B#compaction#32 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:57:43,064 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/b7009c14f6814fd9acece2ad30bf0e99 is 50, key is test_row_0/B:col10/1732539462301/Put/seqid=0 2024-11-25T12:57:43,088 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-25T12:57:43,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:43,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. as already flushing 2024-11-25T12:57:43,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:43,089 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:43,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:43,095 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/87e70f098c944995afe1d7c489f002bc is 50, key is test_row_0/A:col10/1732539463048/Put/seqid=0 2024-11-25T12:57:43,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:43,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741869_1045 (size=12493) 2024-11-25T12:57:43,164 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/b7009c14f6814fd9acece2ad30bf0e99 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/b7009c14f6814fd9acece2ad30bf0e99 2024-11-25T12:57:43,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741870_1046 (size=12151) 2024-11-25T12:57:43,186 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/87e70f098c944995afe1d7c489f002bc 2024-11-25T12:57:43,228 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/030a9fe4f794449da3ba4f0d83c8f7ee is 50, key is test_row_0/B:col10/1732539463048/Put/seqid=0 2024-11-25T12:57:43,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:43,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54986 deadline: 1732539523183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:43,230 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:43,230 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:43,230 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:43,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:43,240 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e7fe900c90851bb165c5d4e631eedb85/B of e7fe900c90851bb165c5d4e631eedb85 into b7009c14f6814fd9acece2ad30bf0e99(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:57:43,240 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7fe900c90851bb165c5d4e631eedb85: 2024-11-25T12:57:43,240 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., storeName=e7fe900c90851bb165c5d4e631eedb85/B, priority=12, startTime=1732539462823; duration=0sec 2024-11-25T12:57:43,240 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:57:43,240 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7fe900c90851bb165c5d4e631eedb85:B 2024-11-25T12:57:43,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54984 deadline: 1732539523231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:43,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:43,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:43,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:43,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:43,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:43,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-25T12:57:43,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55028 deadline: 1732539523231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:43,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:43,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:43,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:43,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55004 deadline: 1732539523233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:43,253 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-25T12:57:43,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:43,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. as already flushing 2024-11-25T12:57:43,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:43,254 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:43,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:43,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:43,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:43,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:43,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:43,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741871_1047 (size=12151) 2024-11-25T12:57:43,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:43,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54986 deadline: 1732539523341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:43,350 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:43,350 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:43,350 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:43,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:43,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54984 deadline: 1732539523350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:43,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:43,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:43,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:43,374 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/880aa15661fd4635988f299f7742698f as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/880aa15661fd4635988f299f7742698f 2024-11-25T12:57:43,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:43,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55028 deadline: 1732539523361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:43,376 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:43,376 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:43,376 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:43,376 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:43,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55004 deadline: 1732539523365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:43,380 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:43,380 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:43,380 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:43,386 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e7fe900c90851bb165c5d4e631eedb85/A of e7fe900c90851bb165c5d4e631eedb85 into 880aa15661fd4635988f299f7742698f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:57:43,387 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7fe900c90851bb165c5d4e631eedb85: 2024-11-25T12:57:43,387 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., storeName=e7fe900c90851bb165c5d4e631eedb85/A, priority=12, startTime=1732539462823; duration=0sec 2024-11-25T12:57:43,388 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:57:43,388 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7fe900c90851bb165c5d4e631eedb85:A 2024-11-25T12:57:43,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:43,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54990 deadline: 1732539523381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:43,400 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 1ff8009e463e,35905,1732539449246 for 'test_row_2' in e7fe900c90851bb165c5d4e631eedb85 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2260 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T12:57:43,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:43,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:43,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:43,409 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-25T12:57:43,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:43,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. as already flushing 2024-11-25T12:57:43,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:43,409 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:43,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:43,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:43,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-25T12:57:43,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:43,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. as already flushing 2024-11-25T12:57:43,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:43,566 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:43,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:43,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-25T12:57:43,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:43,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:43,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54984 deadline: 1732539523573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:43,576 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:43,576 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:43,576 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:43,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:43,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54986 deadline: 1732539523573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:43,578 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:43,578 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:43,578 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:43,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:43,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55028 deadline: 1732539523580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:43,590 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:43,590 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:43,590 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:43,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:43,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55004 deadline: 1732539523589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:43,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:43,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:43,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:43,673 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/030a9fe4f794449da3ba4f0d83c8f7ee 2024-11-25T12:57:43,699 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/20821f68503a4213b8be5890d203bcde is 50, key is test_row_0/C:col10/1732539463048/Put/seqid=0 2024-11-25T12:57:43,733 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-25T12:57:43,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:43,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. as already flushing 2024-11-25T12:57:43,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:43,734 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:43,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:43,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:43,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741872_1048 (size=12151) 2024-11-25T12:57:43,749 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/20821f68503a4213b8be5890d203bcde 2024-11-25T12:57:43,758 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/87e70f098c944995afe1d7c489f002bc as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/87e70f098c944995afe1d7c489f002bc 2024-11-25T12:57:43,768 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/87e70f098c944995afe1d7c489f002bc, entries=150, sequenceid=172, filesize=11.9 K 2024-11-25T12:57:43,770 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/030a9fe4f794449da3ba4f0d83c8f7ee as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/030a9fe4f794449da3ba4f0d83c8f7ee 2024-11-25T12:57:43,788 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/030a9fe4f794449da3ba4f0d83c8f7ee, entries=150, sequenceid=172, filesize=11.9 K 2024-11-25T12:57:43,790 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/20821f68503a4213b8be5890d203bcde as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/20821f68503a4213b8be5890d203bcde 2024-11-25T12:57:43,817 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/20821f68503a4213b8be5890d203bcde, entries=150, sequenceid=172, filesize=11.9 K 2024-11-25T12:57:43,820 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for e7fe900c90851bb165c5d4e631eedb85 in 767ms, sequenceid=172, compaction requested=false 2024-11-25T12:57:43,820 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e7fe900c90851bb165c5d4e631eedb85: 2024-11-25T12:57:43,890 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-25T12:57:43,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:43,890 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing e7fe900c90851bb165c5d4e631eedb85 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-25T12:57:43,912 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. as already flushing 2024-11-25T12:57:43,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on e7fe900c90851bb165c5d4e631eedb85 2024-11-25T12:57:43,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/4df90064a0c94f25b820c9e377429e2f is 50, key is test_row_0/A:col10/1732539463206/Put/seqid=0 2024-11-25T12:57:43,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741873_1049 (size=14541) 2024-11-25T12:57:43,968 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/4df90064a0c94f25b820c9e377429e2f 2024-11-25T12:57:43,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:43,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/4c0473c5424f46a99eb53620d2a31bda is 50, key is test_row_0/B:col10/1732539463206/Put/seqid=0 2024-11-25T12:57:43,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54986 deadline: 1732539523937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:43,998 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:43,998 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:43,998 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:44,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:44,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54984 deadline: 1732539523996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:44,005 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:44,005 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:44,006 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:44,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:44,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55028 deadline: 1732539523997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:44,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:44,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:44,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:44,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55004 deadline: 1732539523998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:44,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:44,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:44,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:44,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:44,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741874_1050 (size=12151) 2024-11-25T12:57:44,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-25T12:57:44,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:44,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54986 deadline: 1732539524100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:44,108 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:44,108 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:44,108 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:44,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:44,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54984 deadline: 1732539524110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:44,116 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:44,117 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:44,117 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:44,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:44,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55028 deadline: 1732539524110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:44,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:44,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:44,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:44,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:44,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55004 deadline: 1732539524113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:44,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:44,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:44,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:44,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:44,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54984 deadline: 1732539524320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:44,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:44,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:44,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:44,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:44,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:44,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55028 deadline: 1732539524330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:44,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:44,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:44,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:44,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:44,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55004 deadline: 1732539524331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:44,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:44,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:44,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:44,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54986 deadline: 1732539524320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:44,339 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:44,340 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:44,340 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:44,428 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/4c0473c5424f46a99eb53620d2a31bda 2024-11-25T12:57:44,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/197d05894e1b4f19996bda24406dae7f is 50, key is test_row_0/C:col10/1732539463206/Put/seqid=0 2024-11-25T12:57:44,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741875_1051 (size=12151) 2024-11-25T12:57:44,512 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/197d05894e1b4f19996bda24406dae7f 2024-11-25T12:57:44,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/4df90064a0c94f25b820c9e377429e2f as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/4df90064a0c94f25b820c9e377429e2f 2024-11-25T12:57:44,550 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/4df90064a0c94f25b820c9e377429e2f, entries=200, sequenceid=200, filesize=14.2 K 2024-11-25T12:57:44,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/4c0473c5424f46a99eb53620d2a31bda as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/4c0473c5424f46a99eb53620d2a31bda 2024-11-25T12:57:44,569 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/4c0473c5424f46a99eb53620d2a31bda, entries=150, sequenceid=200, filesize=11.9 K 2024-11-25T12:57:44,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/197d05894e1b4f19996bda24406dae7f as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/197d05894e1b4f19996bda24406dae7f 2024-11-25T12:57:44,596 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/197d05894e1b4f19996bda24406dae7f, entries=150, sequenceid=200, filesize=11.9 K 2024-11-25T12:57:44,597 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=60.38 KB/61830 for e7fe900c90851bb165c5d4e631eedb85 in 707ms, sequenceid=200, compaction requested=true 2024-11-25T12:57:44,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for e7fe900c90851bb165c5d4e631eedb85: 2024-11-25T12:57:44,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:44,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-25T12:57:44,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-25T12:57:44,606 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-25T12:57:44,606 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6740 sec 2024-11-25T12:57:44,614 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.6850 sec 2024-11-25T12:57:44,649 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:57:44,649 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e7fe900c90851bb165c5d4e631eedb85 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-25T12:57:44,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on e7fe900c90851bb165c5d4e631eedb85 2024-11-25T12:57:44,669 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/5d386441780d4718bb4660845cd830e8 is 50, key is test_row_0/A:col10/1732539464644/Put/seqid=0 2024-11-25T12:57:44,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741876_1052 (size=16931) 2024-11-25T12:57:44,734 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/5d386441780d4718bb4660845cd830e8 2024-11-25T12:57:44,758 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/28cc338e22254dd8948534e4463c0953 is 50, key is test_row_0/B:col10/1732539464644/Put/seqid=0 2024-11-25T12:57:44,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:44,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55028 deadline: 1732539524756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:44,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:44,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54984 deadline: 1732539524762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:44,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:44,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55004 deadline: 1732539524776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:44,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:44,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:44,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:44,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:44,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:44,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:44,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:44,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:44,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:44,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:44,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54986 deadline: 1732539524779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:44,790 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:44,790 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:44,790 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:44,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741877_1053 (size=12151) 2024-11-25T12:57:44,805 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/28cc338e22254dd8948534e4463c0953 2024-11-25T12:57:44,832 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/fb05acba06d64a4696e0e59c3182b872 is 50, key is test_row_0/C:col10/1732539464644/Put/seqid=0 2024-11-25T12:57:44,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741878_1054 (size=12151) 2024-11-25T12:57:44,880 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/fb05acba06d64a4696e0e59c3182b872 2024-11-25T12:57:44,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:44,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55028 deadline: 1732539524890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:44,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:44,895 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:44,895 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:44,902 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/5d386441780d4718bb4660845cd830e8 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/5d386441780d4718bb4660845cd830e8 2024-11-25T12:57:44,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:44,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55004 deadline: 1732539524894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:44,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:44,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54984 deadline: 1732539524901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:44,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:44,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54986 deadline: 1732539524901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:44,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:44,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:44,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:44,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:44,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:44,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:44,912 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:44,912 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:44,913 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:44,918 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/5d386441780d4718bb4660845cd830e8, entries=250, sequenceid=214, filesize=16.5 K 2024-11-25T12:57:44,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/28cc338e22254dd8948534e4463c0953 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/28cc338e22254dd8948534e4463c0953 2024-11-25T12:57:44,935 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/28cc338e22254dd8948534e4463c0953, entries=150, sequenceid=214, filesize=11.9 K 2024-11-25T12:57:44,938 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/fb05acba06d64a4696e0e59c3182b872 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/fb05acba06d64a4696e0e59c3182b872 2024-11-25T12:57:44,951 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/fb05acba06d64a4696e0e59c3182b872, entries=150, sequenceid=214, filesize=11.9 K 2024-11-25T12:57:44,953 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for e7fe900c90851bb165c5d4e631eedb85 in 304ms, sequenceid=214, compaction requested=true 2024-11-25T12:57:44,953 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e7fe900c90851bb165c5d4e631eedb85: 2024-11-25T12:57:44,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7fe900c90851bb165c5d4e631eedb85:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:57:44,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:57:44,953 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:57:44,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7fe900c90851bb165c5d4e631eedb85:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:57:44,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:57:44,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7fe900c90851bb165c5d4e631eedb85:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:57:44,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-25T12:57:44,953 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:57:44,956 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 56116 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:57:44,957 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1541): e7fe900c90851bb165c5d4e631eedb85/A is initiating minor compaction (all files) 2024-11-25T12:57:44,957 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7fe900c90851bb165c5d4e631eedb85/A in TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:44,957 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/880aa15661fd4635988f299f7742698f, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/87e70f098c944995afe1d7c489f002bc, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/4df90064a0c94f25b820c9e377429e2f, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/5d386441780d4718bb4660845cd830e8] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp, totalSize=54.8 K 2024-11-25T12:57:44,957 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:57:44,957 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): e7fe900c90851bb165c5d4e631eedb85/B is initiating minor compaction (all files) 2024-11-25T12:57:44,957 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7fe900c90851bb165c5d4e631eedb85/B in TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:44,958 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/b7009c14f6814fd9acece2ad30bf0e99, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/030a9fe4f794449da3ba4f0d83c8f7ee, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/4c0473c5424f46a99eb53620d2a31bda, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/28cc338e22254dd8948534e4463c0953] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp, totalSize=47.8 K 2024-11-25T12:57:44,958 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 880aa15661fd4635988f299f7742698f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732539461112 2024-11-25T12:57:44,958 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting b7009c14f6814fd9acece2ad30bf0e99, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732539461112 2024-11-25T12:57:44,959 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87e70f098c944995afe1d7c489f002bc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732539462353 2024-11-25T12:57:44,959 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 030a9fe4f794449da3ba4f0d83c8f7ee, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732539462353 2024-11-25T12:57:44,959 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4df90064a0c94f25b820c9e377429e2f, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1732539463147 2024-11-25T12:57:44,960 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c0473c5424f46a99eb53620d2a31bda, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1732539463206 2024-11-25T12:57:44,960 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d386441780d4718bb4660845cd830e8, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732539463937 2024-11-25T12:57:44,960 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 28cc338e22254dd8948534e4463c0953, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732539463945 2024-11-25T12:57:44,983 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7fe900c90851bb165c5d4e631eedb85#B#compaction#42 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:57:44,984 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/798a4005e9d045c4a16828210990a32e is 50, key is test_row_0/B:col10/1732539464644/Put/seqid=0 2024-11-25T12:57:45,002 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7fe900c90851bb165c5d4e631eedb85#A#compaction#43 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:57:45,003 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/1e054310565b4cc39c7e4ac67e31422a is 50, key is test_row_0/A:col10/1732539464644/Put/seqid=0 2024-11-25T12:57:45,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741879_1055 (size=12629) 2024-11-25T12:57:45,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741880_1056 (size=12629) 2024-11-25T12:57:45,089 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/1e054310565b4cc39c7e4ac67e31422a as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/1e054310565b4cc39c7e4ac67e31422a 2024-11-25T12:57:45,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-25T12:57:45,091 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:57:45,092 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:57:45,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-25T12:57:45,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-25T12:57:45,095 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:57:45,099 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:57:45,099 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:57:45,102 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e7fe900c90851bb165c5d4e631eedb85/A of e7fe900c90851bb165c5d4e631eedb85 into 1e054310565b4cc39c7e4ac67e31422a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:57:45,102 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7fe900c90851bb165c5d4e631eedb85: 2024-11-25T12:57:45,102 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., storeName=e7fe900c90851bb165c5d4e631eedb85/A, priority=12, startTime=1732539464953; duration=0sec 2024-11-25T12:57:45,102 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:57:45,102 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7fe900c90851bb165c5d4e631eedb85:A 2024-11-25T12:57:45,102 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:57:45,106 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:57:45,106 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1541): e7fe900c90851bb165c5d4e631eedb85/C is initiating minor compaction (all files) 2024-11-25T12:57:45,106 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7fe900c90851bb165c5d4e631eedb85/C in TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:45,106 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/26ccdd28c269477886a50316129ed767, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/20821f68503a4213b8be5890d203bcde, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/197d05894e1b4f19996bda24406dae7f, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/fb05acba06d64a4696e0e59c3182b872] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp, totalSize=47.8 K 2024-11-25T12:57:45,107 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26ccdd28c269477886a50316129ed767, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732539461112 2024-11-25T12:57:45,108 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:57:45,108 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e7fe900c90851bb165c5d4e631eedb85 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-25T12:57:45,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on e7fe900c90851bb165c5d4e631eedb85 2024-11-25T12:57:45,117 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20821f68503a4213b8be5890d203bcde, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732539462353 2024-11-25T12:57:45,126 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 197d05894e1b4f19996bda24406dae7f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1732539463206 2024-11-25T12:57:45,128 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb05acba06d64a4696e0e59c3182b872, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732539463945 2024-11-25T12:57:45,130 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/e487bcf33fa44e7d91432d6c5319f2c9 is 50, key is test_row_0/A:col10/1732539465103/Put/seqid=0 2024-11-25T12:57:45,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:45,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55004 deadline: 1732539525151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:45,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:45,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:45,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:45,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:45,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55028 deadline: 1732539525162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:45,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:45,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:45,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:45,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:45,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54986 deadline: 1732539525164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:45,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:45,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54984 deadline: 1732539525164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:45,172 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:45,172 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:45,172 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:45,178 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7fe900c90851bb165c5d4e631eedb85#C#compaction#45 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:57:45,179 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/f28a6a68d93a4e9580e0a282070e1c78 is 50, key is test_row_0/C:col10/1732539464644/Put/seqid=0 2024-11-25T12:57:45,179 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:45,179 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:45,179 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:45,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741881_1057 (size=14541) 2024-11-25T12:57:45,191 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/e487bcf33fa44e7d91432d6c5319f2c9 2024-11-25T12:57:45,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-25T12:57:45,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741882_1058 (size=12629) 2024-11-25T12:57:45,232 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/57e656f92c2b44b3bcf31b011141313a is 50, key is test_row_0/B:col10/1732539465103/Put/seqid=0 2024-11-25T12:57:45,253 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-25T12:57:45,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:45,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. as already flushing 2024-11-25T12:57:45,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:45,254 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:45,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:45,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:45,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:45,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55028 deadline: 1732539525280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:45,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:45,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:45,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:45,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:45,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55004 deadline: 1732539525281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:45,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:45,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:45,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54984 deadline: 1732539525281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:45,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:45,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:45,289 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:45,289 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:45,289 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:45,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741883_1059 (size=12151) 2024-11-25T12:57:45,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:45,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54986 deadline: 1732539525291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:45,315 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:45,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:45,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:45,412 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-25T12:57:45,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:45,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. as already flushing 2024-11-25T12:57:45,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:45,416 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:45,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:45,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:45,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-25T12:57:45,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:45,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54990 deadline: 1732539525421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:45,436 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 1ff8009e463e,35905,1732539449246 for 'test_row_2' in e7fe900c90851bb165c5d4e631eedb85 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4300 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T12:57:45,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:45,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:45,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:45,463 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/798a4005e9d045c4a16828210990a32e as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/798a4005e9d045c4a16828210990a32e 2024-11-25T12:57:45,489 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e7fe900c90851bb165c5d4e631eedb85/B of e7fe900c90851bb165c5d4e631eedb85 into 798a4005e9d045c4a16828210990a32e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:57:45,489 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7fe900c90851bb165c5d4e631eedb85: 2024-11-25T12:57:45,489 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., storeName=e7fe900c90851bb165c5d4e631eedb85/B, priority=12, startTime=1732539464953; duration=0sec 2024-11-25T12:57:45,489 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:57:45,490 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7fe900c90851bb165c5d4e631eedb85:B 2024-11-25T12:57:45,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:45,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55028 deadline: 1732539525490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:45,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:45,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:45,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:45,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:45,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55004 deadline: 1732539525501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:45,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:45,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54984 deadline: 1732539525501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:45,513 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:45,513 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:45,513 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:45,513 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:45,513 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:45,514 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:45,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:45,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54986 deadline: 1732539525520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:45,529 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:45,529 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:45,529 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:45,570 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-25T12:57:45,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:45,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. as already flushing 2024-11-25T12:57:45,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:45,571 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:45,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:45,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:45,634 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/f28a6a68d93a4e9580e0a282070e1c78 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/f28a6a68d93a4e9580e0a282070e1c78 2024-11-25T12:57:45,645 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e7fe900c90851bb165c5d4e631eedb85/C of e7fe900c90851bb165c5d4e631eedb85 into f28a6a68d93a4e9580e0a282070e1c78(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:57:45,646 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7fe900c90851bb165c5d4e631eedb85: 2024-11-25T12:57:45,646 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., storeName=e7fe900c90851bb165c5d4e631eedb85/C, priority=12, startTime=1732539464953; duration=0sec 2024-11-25T12:57:45,646 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:57:45,646 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7fe900c90851bb165c5d4e631eedb85:C 2024-11-25T12:57:45,705 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/57e656f92c2b44b3bcf31b011141313a 2024-11-25T12:57:45,725 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/550f6c82dcd341428966a9c43fd1e1f5 is 50, key is test_row_0/C:col10/1732539465103/Put/seqid=0 2024-11-25T12:57:45,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-25T12:57:45,732 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-25T12:57:45,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:45,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. as already flushing 2024-11-25T12:57:45,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:45,733 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:45,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:45,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741884_1060 (size=12151) 2024-11-25T12:57:45,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:45,736 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/550f6c82dcd341428966a9c43fd1e1f5 2024-11-25T12:57:45,744 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/e487bcf33fa44e7d91432d6c5319f2c9 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/e487bcf33fa44e7d91432d6c5319f2c9 2024-11-25T12:57:45,761 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/e487bcf33fa44e7d91432d6c5319f2c9, entries=200, sequenceid=238, filesize=14.2 K 2024-11-25T12:57:45,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/57e656f92c2b44b3bcf31b011141313a as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/57e656f92c2b44b3bcf31b011141313a 2024-11-25T12:57:45,769 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/57e656f92c2b44b3bcf31b011141313a, entries=150, sequenceid=238, filesize=11.9 K 2024-11-25T12:57:45,771 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/550f6c82dcd341428966a9c43fd1e1f5 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/550f6c82dcd341428966a9c43fd1e1f5 2024-11-25T12:57:45,789 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/550f6c82dcd341428966a9c43fd1e1f5, entries=150, sequenceid=238, filesize=11.9 K 2024-11-25T12:57:45,790 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for e7fe900c90851bb165c5d4e631eedb85 in 682ms, sequenceid=238, compaction requested=false 2024-11-25T12:57:45,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e7fe900c90851bb165c5d4e631eedb85: 2024-11-25T12:57:45,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on e7fe900c90851bb165c5d4e631eedb85 2024-11-25T12:57:45,814 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:57:45,815 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e7fe900c90851bb165c5d4e631eedb85 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-25T12:57:45,833 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/791057c8351941b6a807b408f2a0d253 is 50, key is test_row_0/A:col10/1732539465134/Put/seqid=0 2024-11-25T12:57:45,887 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-25T12:57:45,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:45,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. as already flushing 2024-11-25T12:57:45,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:45,887 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:45,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:45,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:45,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741885_1061 (size=14541) 2024-11-25T12:57:45,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:45,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:45,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55028 deadline: 1732539525887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:45,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54986 deadline: 1732539525887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:45,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:45,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:45,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:45,920 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:45,920 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:45,920 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:45,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:45,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54984 deadline: 1732539525919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:45,924 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:45,924 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:45,924 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:45,924 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:45,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55004 deadline: 1732539525920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:45,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:45,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:45,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:46,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:46,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54984 deadline: 1732539526031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:46,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:46,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:46,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:46,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:46,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55004 deadline: 1732539526031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:46,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:46,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:46,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:46,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:46,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55028 deadline: 1732539526031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:46,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:46,034 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:46,034 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:46,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:46,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54986 deadline: 1732539526031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:46,037 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:46,037 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:46,037 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:46,048 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-25T12:57:46,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:46,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. as already flushing 2024-11-25T12:57:46,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:46,048 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:46,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:46,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:46,202 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-25T12:57:46,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:46,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. as already flushing 2024-11-25T12:57:46,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:46,202 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:46,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:46,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:46,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-25T12:57:46,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:46,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54986 deadline: 1732539526241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:46,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:46,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55028 deadline: 1732539526242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:46,256 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:46,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:46,256 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:46,256 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:46,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54984 deadline: 1732539526243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:46,257 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:46,257 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:46,257 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:46,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:46,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:46,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:46,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:46,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55004 deadline: 1732539526245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:46,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:46,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:46,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:46,296 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/791057c8351941b6a807b408f2a0d253 2024-11-25T12:57:46,309 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/fa73283293ac4ecbbe3f1b2f7d344f59 is 50, key is test_row_0/B:col10/1732539465134/Put/seqid=0 2024-11-25T12:57:46,356 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-25T12:57:46,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:46,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. as already flushing 2024-11-25T12:57:46,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:46,356 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:46,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:46,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:46,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741886_1062 (size=12151) 2024-11-25T12:57:46,376 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/fa73283293ac4ecbbe3f1b2f7d344f59 2024-11-25T12:57:46,391 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/73923858f942427f8f83639ab50ab7d7 is 50, key is test_row_0/C:col10/1732539465134/Put/seqid=0 2024-11-25T12:57:46,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741887_1063 (size=12151) 2024-11-25T12:57:46,513 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-25T12:57:46,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:46,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. as already flushing 2024-11-25T12:57:46,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:46,514 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:46,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:46,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:46,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54986 deadline: 1732539526560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:46,564 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:46,564 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:46,564 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:46,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:46,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:46,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54984 deadline: 1732539526574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:46,583 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:46,583 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:46,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:46,583 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:46,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55028 deadline: 1732539526574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:46,585 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:46,585 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:46,585 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:46,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:46,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55004 deadline: 1732539526580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:46,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:46,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:46,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:46,723 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-25T12:57:46,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:46,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. as already flushing 2024-11-25T12:57:46,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:46,723 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:46,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:46,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:46,817 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/73923858f942427f8f83639ab50ab7d7 2024-11-25T12:57:46,858 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/791057c8351941b6a807b408f2a0d253 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/791057c8351941b6a807b408f2a0d253 2024-11-25T12:57:46,875 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/791057c8351941b6a807b408f2a0d253, entries=200, sequenceid=254, filesize=14.2 K 2024-11-25T12:57:46,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/fa73283293ac4ecbbe3f1b2f7d344f59 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/fa73283293ac4ecbbe3f1b2f7d344f59 2024-11-25T12:57:46,887 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/fa73283293ac4ecbbe3f1b2f7d344f59, entries=150, sequenceid=254, filesize=11.9 K 2024-11-25T12:57:46,891 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/73923858f942427f8f83639ab50ab7d7 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/73923858f942427f8f83639ab50ab7d7 2024-11-25T12:57:46,891 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-25T12:57:46,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:46,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. as already flushing 2024-11-25T12:57:46,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:46,892 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:46,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:46,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:46,930 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/73923858f942427f8f83639ab50ab7d7, entries=150, sequenceid=254, filesize=11.9 K 2024-11-25T12:57:46,934 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for e7fe900c90851bb165c5d4e631eedb85 in 1120ms, sequenceid=254, compaction requested=true 2024-11-25T12:57:46,934 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e7fe900c90851bb165c5d4e631eedb85: 2024-11-25T12:57:46,934 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:57:46,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7fe900c90851bb165c5d4e631eedb85:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:57:46,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:57:46,935 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:57:46,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7fe900c90851bb165c5d4e631eedb85:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:57:46,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:57:46,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7fe900c90851bb165c5d4e631eedb85:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:57:46,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:57:46,938 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:57:46,939 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1541): e7fe900c90851bb165c5d4e631eedb85/A is initiating minor compaction (all files) 2024-11-25T12:57:46,939 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7fe900c90851bb165c5d4e631eedb85/A in TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:46,939 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/1e054310565b4cc39c7e4ac67e31422a, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/e487bcf33fa44e7d91432d6c5319f2c9, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/791057c8351941b6a807b408f2a0d253] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp, totalSize=40.7 K 2024-11-25T12:57:46,939 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:57:46,939 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): e7fe900c90851bb165c5d4e631eedb85/B is initiating minor compaction (all files) 2024-11-25T12:57:46,939 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7fe900c90851bb165c5d4e631eedb85/B in TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:46,939 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/798a4005e9d045c4a16828210990a32e, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/57e656f92c2b44b3bcf31b011141313a, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/fa73283293ac4ecbbe3f1b2f7d344f59] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp, totalSize=36.1 K 2024-11-25T12:57:46,939 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e054310565b4cc39c7e4ac67e31422a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732539463945 2024-11-25T12:57:46,940 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 798a4005e9d045c4a16828210990a32e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732539463945 2024-11-25T12:57:46,940 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 57e656f92c2b44b3bcf31b011141313a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1732539464759 2024-11-25T12:57:46,940 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting e487bcf33fa44e7d91432d6c5319f2c9, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1732539464759 2024-11-25T12:57:46,941 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting fa73283293ac4ecbbe3f1b2f7d344f59, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732539465134 2024-11-25T12:57:46,941 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 791057c8351941b6a807b408f2a0d253, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732539465134 2024-11-25T12:57:46,970 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7fe900c90851bb165c5d4e631eedb85#B#compaction#51 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:57:46,971 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/37f6f3364c5f413d9024da547a427fdb is 50, key is test_row_0/B:col10/1732539465134/Put/seqid=0 2024-11-25T12:57:46,980 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7fe900c90851bb165c5d4e631eedb85#A#compaction#52 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:57:46,980 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/12e4a0449da246319b5e69c735feab2d is 50, key is test_row_0/A:col10/1732539465134/Put/seqid=0 2024-11-25T12:57:47,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741888_1064 (size=12731) 2024-11-25T12:57:47,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741889_1065 (size=12731) 2024-11-25T12:57:47,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-25T12:57:47,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:47,045 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing e7fe900c90851bb165c5d4e631eedb85 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-25T12:57:47,051 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/37f6f3364c5f413d9024da547a427fdb as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/37f6f3364c5f413d9024da547a427fdb 2024-11-25T12:57:47,052 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/12e4a0449da246319b5e69c735feab2d as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/12e4a0449da246319b5e69c735feab2d 2024-11-25T12:57:47,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/261e9ca73b73459eb1fdbb8f8a43efce is 50, key is test_row_0/A:col10/1732539465877/Put/seqid=0 2024-11-25T12:57:47,064 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7fe900c90851bb165c5d4e631eedb85/B of e7fe900c90851bb165c5d4e631eedb85 into 37f6f3364c5f413d9024da547a427fdb(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:57:47,064 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7fe900c90851bb165c5d4e631eedb85: 2024-11-25T12:57:47,064 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7fe900c90851bb165c5d4e631eedb85/A of e7fe900c90851bb165c5d4e631eedb85 into 12e4a0449da246319b5e69c735feab2d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:57:47,064 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., storeName=e7fe900c90851bb165c5d4e631eedb85/B, priority=13, startTime=1732539466935; duration=0sec 2024-11-25T12:57:47,064 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7fe900c90851bb165c5d4e631eedb85: 2024-11-25T12:57:47,064 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., storeName=e7fe900c90851bb165c5d4e631eedb85/A, priority=13, startTime=1732539466934; duration=0sec 2024-11-25T12:57:47,064 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:57:47,065 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7fe900c90851bb165c5d4e631eedb85:B 2024-11-25T12:57:47,065 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:57:47,066 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:57:47,066 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7fe900c90851bb165c5d4e631eedb85:A 2024-11-25T12:57:47,069 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:57:47,069 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): e7fe900c90851bb165c5d4e631eedb85/C is initiating minor compaction (all files) 2024-11-25T12:57:47,069 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7fe900c90851bb165c5d4e631eedb85/C in TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:47,069 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/f28a6a68d93a4e9580e0a282070e1c78, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/550f6c82dcd341428966a9c43fd1e1f5, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/73923858f942427f8f83639ab50ab7d7] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp, totalSize=36.1 K 2024-11-25T12:57:47,070 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting f28a6a68d93a4e9580e0a282070e1c78, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732539463945 2024-11-25T12:57:47,071 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 550f6c82dcd341428966a9c43fd1e1f5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1732539464759 2024-11-25T12:57:47,072 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 73923858f942427f8f83639ab50ab7d7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732539465134 2024-11-25T12:57:47,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on e7fe900c90851bb165c5d4e631eedb85 2024-11-25T12:57:47,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. as already flushing 2024-11-25T12:57:47,101 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7fe900c90851bb165c5d4e631eedb85#C#compaction#54 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:57:47,102 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/8c3daa5bdbda48b0b247bcbee49493e8 is 50, key is test_row_0/C:col10/1732539465134/Put/seqid=0 2024-11-25T12:57:47,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741890_1066 (size=12301) 2024-11-25T12:57:47,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:47,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54984 deadline: 1732539527114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:47,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:47,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:47,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:47,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:47,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55028 deadline: 1732539527116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:47,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:47,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:47,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:47,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:47,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:47,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55004 deadline: 1732539527119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:47,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54986 deadline: 1732539527115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:47,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:47,124 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:47,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:47,124 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:47,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:47,125 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:47,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741891_1067 (size=12731) 2024-11-25T12:57:47,155 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/8c3daa5bdbda48b0b247bcbee49493e8 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/8c3daa5bdbda48b0b247bcbee49493e8 2024-11-25T12:57:47,165 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7fe900c90851bb165c5d4e631eedb85/C of e7fe900c90851bb165c5d4e631eedb85 into 8c3daa5bdbda48b0b247bcbee49493e8(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:57:47,165 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7fe900c90851bb165c5d4e631eedb85: 2024-11-25T12:57:47,165 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., storeName=e7fe900c90851bb165c5d4e631eedb85/C, priority=13, startTime=1732539466935; duration=0sec 2024-11-25T12:57:47,165 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:57:47,165 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7fe900c90851bb165c5d4e631eedb85:C 2024-11-25T12:57:47,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:47,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:47,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55028 deadline: 1732539527230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:47,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55004 deadline: 1732539527230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:47,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:47,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:47,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:47,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:47,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:47,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:47,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:47,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54984 deadline: 1732539527230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:47,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:47,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54986 deadline: 1732539527230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:47,233 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:47,233 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:47,233 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:47,235 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:47,235 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:47,236 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:47,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-25T12:57:47,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:47,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:47,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55028 deadline: 1732539527440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:47,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:47,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:47,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:47,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54984 deadline: 1732539527440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:47,446 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:47,446 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:47,446 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:47,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:47,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55004 deadline: 1732539527442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:47,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:47,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:47,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:47,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:47,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54986 deadline: 1732539527443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:47,451 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:47,451 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:47,452 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:47,507 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/261e9ca73b73459eb1fdbb8f8a43efce 2024-11-25T12:57:47,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/6defa8c81c2d4daab663ed5b4247cfc9 is 50, key is test_row_0/B:col10/1732539465877/Put/seqid=0 2024-11-25T12:57:47,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741892_1068 (size=12301) 2024-11-25T12:57:47,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:47,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:47,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55004 deadline: 1732539527750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:47,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:47,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:47,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:47,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:47,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55028 deadline: 1732539527750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:47,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54984 deadline: 1732539527750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:47,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:47,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:47,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:47,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:47,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:47,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:47,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:47,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54986 deadline: 1732539527760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:47,765 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:47,765 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:47,765 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:47,988 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/6defa8c81c2d4daab663ed5b4247cfc9 2024-11-25T12:57:48,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/29df4b2f6c7741538af70d5bf16a7fba is 50, key is test_row_0/C:col10/1732539465877/Put/seqid=0 2024-11-25T12:57:48,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741893_1069 (size=12301) 2024-11-25T12:57:48,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:48,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55004 deadline: 1732539528270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:48,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:48,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:48,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:48,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:48,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54984 deadline: 1732539528270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:48,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55028 deadline: 1732539528270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:48,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:48,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:48,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:48,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:48,273 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:48,273 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:48,273 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:48,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:48,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54986 deadline: 1732539528280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:48,282 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:48,282 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:48,282 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:48,470 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/29df4b2f6c7741538af70d5bf16a7fba 2024-11-25T12:57:48,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/261e9ca73b73459eb1fdbb8f8a43efce as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/261e9ca73b73459eb1fdbb8f8a43efce 2024-11-25T12:57:48,488 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/261e9ca73b73459eb1fdbb8f8a43efce, entries=150, sequenceid=276, filesize=12.0 K 2024-11-25T12:57:48,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/6defa8c81c2d4daab663ed5b4247cfc9 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/6defa8c81c2d4daab663ed5b4247cfc9 2024-11-25T12:57:48,496 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/6defa8c81c2d4daab663ed5b4247cfc9, entries=150, sequenceid=276, filesize=12.0 K 2024-11-25T12:57:48,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/29df4b2f6c7741538af70d5bf16a7fba as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/29df4b2f6c7741538af70d5bf16a7fba 2024-11-25T12:57:48,537 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/29df4b2f6c7741538af70d5bf16a7fba, entries=150, sequenceid=276, filesize=12.0 K 2024-11-25T12:57:48,538 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for e7fe900c90851bb165c5d4e631eedb85 in 1493ms, sequenceid=276, compaction requested=false 2024-11-25T12:57:48,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for e7fe900c90851bb165c5d4e631eedb85: 2024-11-25T12:57:48,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:48,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-25T12:57:48,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-25T12:57:48,544 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-25T12:57:48,544 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.4420 sec 2024-11-25T12:57:48,550 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 3.4550 sec 2024-11-25T12:57:49,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-25T12:57:49,283 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:57:49,284 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:57:49,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-25T12:57:49,288 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:57:49,290 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:57:49,290 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:57:49,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-25T12:57:49,295 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:57:49,296 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e7fe900c90851bb165c5d4e631eedb85 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-25T12:57:49,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on e7fe900c90851bb165c5d4e631eedb85 2024-11-25T12:57:49,318 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/ec9a7dbf37214cfd9d5e3403864eff5d is 50, key is test_row_0/A:col10/1732539467116/Put/seqid=0 2024-11-25T12:57:49,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741894_1070 (size=12301) 2024-11-25T12:57:49,375 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/ec9a7dbf37214cfd9d5e3403864eff5d 2024-11-25T12:57:49,403 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/8cbfe687d6b1411e89b5f01a7910d877 is 50, key is test_row_0/B:col10/1732539467116/Put/seqid=0 2024-11-25T12:57:49,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-25T12:57:49,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:49,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54986 deadline: 1732539529400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:49,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:49,413 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:49,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54984 deadline: 1732539529401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:49,413 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:49,413 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:49,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:49,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:49,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:49,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:49,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55028 deadline: 1732539529417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:49,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:49,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:49,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:49,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:49,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55004 deadline: 1732539529417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:49,425 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:49,425 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:49,425 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:49,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741895_1071 (size=12301) 2024-11-25T12:57:49,453 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-25T12:57:49,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:49,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. as already flushing 2024-11-25T12:57:49,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:49,454 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:49,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:49,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:49,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:49,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54990 deadline: 1732539529470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:49,474 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 1ff8009e463e,35905,1732539449246 for 'test_row_2' in e7fe900c90851bb165c5d4e631eedb85 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8337 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T12:57:49,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:49,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:49,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:49,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:49,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54986 deadline: 1732539529520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:49,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:49,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54984 deadline: 1732539529520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:49,523 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:49,523 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:49,523 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:49,524 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:49,524 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:49,524 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:49,533 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:49,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55004 deadline: 1732539529531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:49,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:49,534 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:49,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:49,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:49,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55028 deadline: 1732539529531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:49,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:49,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:49,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:49,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-25T12:57:49,611 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-25T12:57:49,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:49,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. as already flushing 2024-11-25T12:57:49,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:49,611 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:49,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:49,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:49,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:49,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54984 deadline: 1732539529730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:49,732 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:49,732 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:49,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:49,732 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:49,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54986 deadline: 1732539529731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:49,732 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:49,732 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:49,732 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:49,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:49,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55028 deadline: 1732539529740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:49,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:49,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55004 deadline: 1732539529741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:49,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:49,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:49,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:49,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:49,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:49,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:49,765 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-25T12:57:49,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:49,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. as already flushing 2024-11-25T12:57:49,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:49,766 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:49,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:49,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:49,845 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/8cbfe687d6b1411e89b5f01a7910d877 2024-11-25T12:57:49,867 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/50bf4f0c88b448f18d515cde53083631 is 50, key is test_row_0/C:col10/1732539467116/Put/seqid=0 2024-11-25T12:57:49,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741896_1072 (size=12301) 2024-11-25T12:57:49,902 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/50bf4f0c88b448f18d515cde53083631 2024-11-25T12:57:49,920 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-25T12:57:49,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:49,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. as already flushing 2024-11-25T12:57:49,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-25T12:57:49,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:49,920 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:49,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:49,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:49,938 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/ec9a7dbf37214cfd9d5e3403864eff5d as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/ec9a7dbf37214cfd9d5e3403864eff5d 2024-11-25T12:57:49,947 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/ec9a7dbf37214cfd9d5e3403864eff5d, entries=150, sequenceid=295, filesize=12.0 K 2024-11-25T12:57:49,953 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/8cbfe687d6b1411e89b5f01a7910d877 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/8cbfe687d6b1411e89b5f01a7910d877 2024-11-25T12:57:49,973 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/8cbfe687d6b1411e89b5f01a7910d877, entries=150, sequenceid=295, filesize=12.0 K 2024-11-25T12:57:49,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/50bf4f0c88b448f18d515cde53083631 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/50bf4f0c88b448f18d515cde53083631 2024-11-25T12:57:50,007 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/50bf4f0c88b448f18d515cde53083631, entries=150, sequenceid=295, filesize=12.0 K 2024-11-25T12:57:50,009 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for e7fe900c90851bb165c5d4e631eedb85 in 714ms, sequenceid=295, compaction requested=true 2024-11-25T12:57:50,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e7fe900c90851bb165c5d4e631eedb85: 2024-11-25T12:57:50,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7fe900c90851bb165c5d4e631eedb85:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:57:50,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:57:50,010 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:57:50,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7fe900c90851bb165c5d4e631eedb85:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:57:50,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:57:50,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7fe900c90851bb165c5d4e631eedb85:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:57:50,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-25T12:57:50,010 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:57:50,011 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:57:50,012 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1541): e7fe900c90851bb165c5d4e631eedb85/A is initiating minor compaction (all files) 2024-11-25T12:57:50,012 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7fe900c90851bb165c5d4e631eedb85/A in TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:50,012 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/12e4a0449da246319b5e69c735feab2d, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/261e9ca73b73459eb1fdbb8f8a43efce, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/ec9a7dbf37214cfd9d5e3403864eff5d] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp, totalSize=36.5 K 2024-11-25T12:57:50,013 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:57:50,013 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12e4a0449da246319b5e69c735feab2d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732539465134 2024-11-25T12:57:50,013 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): e7fe900c90851bb165c5d4e631eedb85/B is initiating minor compaction (all files) 2024-11-25T12:57:50,013 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7fe900c90851bb165c5d4e631eedb85/B in TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:50,013 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/37f6f3364c5f413d9024da547a427fdb, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/6defa8c81c2d4daab663ed5b4247cfc9, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/8cbfe687d6b1411e89b5f01a7910d877] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp, totalSize=36.5 K 2024-11-25T12:57:50,013 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 261e9ca73b73459eb1fdbb8f8a43efce, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732539465874 2024-11-25T12:57:50,013 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 37f6f3364c5f413d9024da547a427fdb, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732539465134 2024-11-25T12:57:50,013 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec9a7dbf37214cfd9d5e3403864eff5d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732539467116 2024-11-25T12:57:50,014 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 6defa8c81c2d4daab663ed5b4247cfc9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732539465874 2024-11-25T12:57:50,015 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 8cbfe687d6b1411e89b5f01a7910d877, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732539467116 2024-11-25T12:57:50,035 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7fe900c90851bb165c5d4e631eedb85#A#compaction#60 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:57:50,036 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/18bbda9e7c904728acb4211c2fe9baf9 is 50, key is test_row_0/A:col10/1732539467116/Put/seqid=0 2024-11-25T12:57:50,051 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:57:50,051 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e7fe900c90851bb165c5d4e631eedb85 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-25T12:57:50,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on e7fe900c90851bb165c5d4e631eedb85 2024-11-25T12:57:50,065 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7fe900c90851bb165c5d4e631eedb85#B#compaction#61 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:57:50,066 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/e1047cb31bc1480f946882e81c535024 is 50, key is test_row_0/B:col10/1732539467116/Put/seqid=0 2024-11-25T12:57:50,085 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-25T12:57:50,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:50,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. as already flushing 2024-11-25T12:57:50,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:50,086 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:50,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:50,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:50,091 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/03724ce6cff245bb86a911abb1492e5b is 50, key is test_row_0/A:col10/1732539469405/Put/seqid=0 2024-11-25T12:57:50,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:50,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55028 deadline: 1732539530090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:50,100 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:50,100 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:50,100 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:50,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:50,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54986 deadline: 1732539530092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:50,101 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:50,101 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:50,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:50,101 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:50,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55004 deadline: 1732539530093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:50,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:50,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:50,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:50,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:50,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54984 deadline: 1732539530096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:50,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:50,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:50,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:50,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741898_1074 (size=12983) 2024-11-25T12:57:50,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741897_1073 (size=12983) 2024-11-25T12:57:50,182 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/e1047cb31bc1480f946882e81c535024 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/e1047cb31bc1480f946882e81c535024 2024-11-25T12:57:50,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741899_1075 (size=14741) 2024-11-25T12:57:50,191 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/03724ce6cff245bb86a911abb1492e5b 2024-11-25T12:57:50,193 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/18bbda9e7c904728acb4211c2fe9baf9 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/18bbda9e7c904728acb4211c2fe9baf9 2024-11-25T12:57:50,194 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7fe900c90851bb165c5d4e631eedb85/B of e7fe900c90851bb165c5d4e631eedb85 into e1047cb31bc1480f946882e81c535024(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:57:50,194 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7fe900c90851bb165c5d4e631eedb85: 2024-11-25T12:57:50,194 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., storeName=e7fe900c90851bb165c5d4e631eedb85/B, priority=13, startTime=1732539470010; duration=0sec 2024-11-25T12:57:50,194 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:57:50,194 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7fe900c90851bb165c5d4e631eedb85:B 2024-11-25T12:57:50,194 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:57:50,198 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:57:50,198 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): e7fe900c90851bb165c5d4e631eedb85/C is initiating minor compaction (all files) 2024-11-25T12:57:50,198 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7fe900c90851bb165c5d4e631eedb85/C in TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:50,198 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/8c3daa5bdbda48b0b247bcbee49493e8, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/29df4b2f6c7741538af70d5bf16a7fba, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/50bf4f0c88b448f18d515cde53083631] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp, totalSize=36.5 K 2024-11-25T12:57:50,201 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c3daa5bdbda48b0b247bcbee49493e8, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732539465134 2024-11-25T12:57:50,205 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 29df4b2f6c7741538af70d5bf16a7fba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732539465874 2024-11-25T12:57:50,206 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7fe900c90851bb165c5d4e631eedb85/A of e7fe900c90851bb165c5d4e631eedb85 into 18bbda9e7c904728acb4211c2fe9baf9(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:57:50,206 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7fe900c90851bb165c5d4e631eedb85: 2024-11-25T12:57:50,206 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., storeName=e7fe900c90851bb165c5d4e631eedb85/A, priority=13, startTime=1732539470010; duration=0sec 2024-11-25T12:57:50,206 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 50bf4f0c88b448f18d515cde53083631, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732539467116 2024-11-25T12:57:50,206 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:57:50,206 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7fe900c90851bb165c5d4e631eedb85:A 2024-11-25T12:57:50,213 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/8b84c83d2e234968bc17de037ff3e1c0 is 50, key is test_row_0/B:col10/1732539469405/Put/seqid=0 2024-11-25T12:57:50,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:50,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54986 deadline: 1732539530210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:50,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:50,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54984 deadline: 1732539530210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:50,220 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:50,220 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:50,221 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:50,221 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:50,221 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:50,221 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:50,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:50,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55004 deadline: 1732539530210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:50,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:50,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:50,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:50,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:50,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55028 deadline: 1732539530209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:50,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:50,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:50,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:50,239 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-25T12:57:50,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:50,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. as already flushing 2024-11-25T12:57:50,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:50,239 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:50,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:50,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:50,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741900_1076 (size=12301) 2024-11-25T12:57:50,265 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/8b84c83d2e234968bc17de037ff3e1c0 2024-11-25T12:57:50,274 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7fe900c90851bb165c5d4e631eedb85#C#compaction#64 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:57:50,275 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/be2ddb98358c4bd5824fd85d13042779 is 50, key is test_row_0/C:col10/1732539467116/Put/seqid=0 2024-11-25T12:57:50,306 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/560e3561e7704047a66074195052ac4b is 50, key is test_row_0/C:col10/1732539469405/Put/seqid=0 2024-11-25T12:57:50,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741901_1077 (size=12983) 2024-11-25T12:57:50,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741902_1078 (size=12301) 2024-11-25T12:57:50,348 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/be2ddb98358c4bd5824fd85d13042779 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/be2ddb98358c4bd5824fd85d13042779 2024-11-25T12:57:50,363 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7fe900c90851bb165c5d4e631eedb85/C of e7fe900c90851bb165c5d4e631eedb85 into be2ddb98358c4bd5824fd85d13042779(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:57:50,363 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7fe900c90851bb165c5d4e631eedb85: 2024-11-25T12:57:50,363 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., storeName=e7fe900c90851bb165c5d4e631eedb85/C, priority=13, startTime=1732539470010; duration=0sec 2024-11-25T12:57:50,363 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:57:50,363 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7fe900c90851bb165c5d4e631eedb85:C 2024-11-25T12:57:50,408 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-25T12:57:50,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:50,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. as already flushing 2024-11-25T12:57:50,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:50,409 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:50,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:50,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:50,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-25T12:57:50,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:50,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55004 deadline: 1732539530461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:50,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:50,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55028 deadline: 1732539530462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:50,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:50,467 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:50,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:50,467 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:50,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:50,467 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:50,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:50,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54986 deadline: 1732539530463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:50,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:50,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54984 deadline: 1732539530463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:50,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:50,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:50,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:50,478 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:50,479 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:50,479 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:50,570 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-25T12:57:50,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:50,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. as already flushing 2024-11-25T12:57:50,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:50,571 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:50,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:50,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:50,733 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-25T12:57:50,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:50,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. as already flushing 2024-11-25T12:57:50,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:50,733 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:50,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:50,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:50,737 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/560e3561e7704047a66074195052ac4b 2024-11-25T12:57:50,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/03724ce6cff245bb86a911abb1492e5b as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/03724ce6cff245bb86a911abb1492e5b 2024-11-25T12:57:50,770 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/03724ce6cff245bb86a911abb1492e5b, entries=200, sequenceid=317, filesize=14.4 K 2024-11-25T12:57:50,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/8b84c83d2e234968bc17de037ff3e1c0 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/8b84c83d2e234968bc17de037ff3e1c0 2024-11-25T12:57:50,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:50,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55028 deadline: 1732539530770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:50,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:50,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:50,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:50,781 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/8b84c83d2e234968bc17de037ff3e1c0, entries=150, sequenceid=317, filesize=12.0 K 2024-11-25T12:57:50,783 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/560e3561e7704047a66074195052ac4b as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/560e3561e7704047a66074195052ac4b 2024-11-25T12:57:50,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:50,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55004 deadline: 1732539530779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:50,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:50,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:50,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:50,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:50,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54984 deadline: 1732539530780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:50,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:50,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:50,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:50,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:50,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54986 deadline: 1732539530790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:50,793 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:50,793 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:50,793 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:50,796 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/560e3561e7704047a66074195052ac4b, entries=150, sequenceid=317, filesize=12.0 K 2024-11-25T12:57:50,797 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for e7fe900c90851bb165c5d4e631eedb85 in 746ms, sequenceid=317, compaction requested=false 2024-11-25T12:57:50,797 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e7fe900c90851bb165c5d4e631eedb85: 2024-11-25T12:57:50,891 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-25T12:57:50,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:50,892 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing e7fe900c90851bb165c5d4e631eedb85 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-25T12:57:50,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/661d0293e834414280cde7fcd4660faa is 50, key is test_row_0/A:col10/1732539470090/Put/seqid=0 2024-11-25T12:57:50,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741903_1079 (size=12301) 2024-11-25T12:57:50,908 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/661d0293e834414280cde7fcd4660faa 2024-11-25T12:57:50,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/c06e1de741124d3cb7eac29e6066d4ed is 50, key is test_row_0/B:col10/1732539470090/Put/seqid=0 2024-11-25T12:57:50,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741904_1080 (size=12301) 2024-11-25T12:57:51,295 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. as already flushing 2024-11-25T12:57:51,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on e7fe900c90851bb165c5d4e631eedb85 2024-11-25T12:57:51,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:51,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54986 deadline: 1732539531351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:51,358 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:51,358 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:51,358 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:51,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:51,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55028 deadline: 1732539531354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:51,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:51,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:51,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:51,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:51,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54984 deadline: 1732539531356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:51,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:51,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:51,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:51,363 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/c06e1de741124d3cb7eac29e6066d4ed 2024-11-25T12:57:51,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:51,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55004 deadline: 1732539531357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:51,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:51,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:51,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:51,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/67c297adf5d044e5bc377ef422495f8b is 50, key is test_row_0/C:col10/1732539470090/Put/seqid=0 2024-11-25T12:57:51,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741905_1081 (size=12301) 2024-11-25T12:57:51,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:51,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54986 deadline: 1732539531460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:51,464 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:51,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:51,464 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:51,464 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:51,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55028 deadline: 1732539531461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:51,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:51,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:51,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:51,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-25T12:57:51,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:51,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:51,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54984 deadline: 1732539531470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:51,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55004 deadline: 1732539531470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:51,476 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:51,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:51,476 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:51,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:51,476 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:51,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:51,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:51,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54986 deadline: 1732539531670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:51,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:51,671 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:51,671 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:51,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55028 deadline: 1732539531670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:51,671 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:51,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:51,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:51,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:51,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:51,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55004 deadline: 1732539531681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:51,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:51,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54984 deadline: 1732539531681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:51,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:51,686 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:51,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:51,686 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:51,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:51,686 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:51,860 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/67c297adf5d044e5bc377ef422495f8b 2024-11-25T12:57:51,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/661d0293e834414280cde7fcd4660faa as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/661d0293e834414280cde7fcd4660faa 2024-11-25T12:57:51,909 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/661d0293e834414280cde7fcd4660faa, entries=150, sequenceid=334, filesize=12.0 K 2024-11-25T12:57:51,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/c06e1de741124d3cb7eac29e6066d4ed as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/c06e1de741124d3cb7eac29e6066d4ed 2024-11-25T12:57:51,930 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/c06e1de741124d3cb7eac29e6066d4ed, entries=150, sequenceid=334, filesize=12.0 K 2024-11-25T12:57:51,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/67c297adf5d044e5bc377ef422495f8b as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/67c297adf5d044e5bc377ef422495f8b 2024-11-25T12:57:51,951 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/67c297adf5d044e5bc377ef422495f8b, entries=150, sequenceid=334, filesize=12.0 K 2024-11-25T12:57:51,964 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for e7fe900c90851bb165c5d4e631eedb85 in 1072ms, sequenceid=334, compaction requested=true 2024-11-25T12:57:51,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for e7fe900c90851bb165c5d4e631eedb85: 2024-11-25T12:57:51,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:51,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-25T12:57:51,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-25T12:57:52,008 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-25T12:57:52,008 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6900 sec 2024-11-25T12:57:52,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on e7fe900c90851bb165c5d4e631eedb85 2024-11-25T12:57:52,009 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:57:52,009 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e7fe900c90851bb165c5d4e631eedb85 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-25T12:57:52,012 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 2.7260 sec 2024-11-25T12:57:52,034 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/f7faea9a44504e94b7cf8d54bf6a1679 is 50, key is test_row_0/A:col10/1732539471350/Put/seqid=0 2024-11-25T12:57:52,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:52,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55028 deadline: 1732539532055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:52,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:52,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:52,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:52,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741906_1082 (size=12301) 2024-11-25T12:57:52,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:52,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54984 deadline: 1732539532056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:52,079 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/f7faea9a44504e94b7cf8d54bf6a1679 2024-11-25T12:57:52,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:52,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:52,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:52,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:52,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54986 deadline: 1732539532065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:52,082 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:52,082 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:52,082 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:52,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:52,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55004 deadline: 1732539532071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:52,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:52,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:52,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:52,147 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/29814e698c324309832c12cacb140da4 is 50, key is test_row_0/B:col10/1732539471350/Put/seqid=0 2024-11-25T12:57:52,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741907_1083 (size=12301) 2024-11-25T12:57:52,183 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/29814e698c324309832c12cacb140da4 2024-11-25T12:57:52,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:52,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55028 deadline: 1732539532180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:52,185 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:52,185 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:52,185 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:52,195 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/82d334f2dd654c87965902044d37535d is 50, key is test_row_0/C:col10/1732539471350/Put/seqid=0 2024-11-25T12:57:52,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:52,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54984 deadline: 1732539532191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:52,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:52,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:52,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:52,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:52,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54986 deadline: 1732539532191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:52,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:52,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55004 deadline: 1732539532191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:52,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:52,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:52,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:52,201 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:52,201 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:52,201 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:52,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741908_1084 (size=12301) 2024-11-25T12:57:52,275 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/82d334f2dd654c87965902044d37535d 2024-11-25T12:57:52,284 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/f7faea9a44504e94b7cf8d54bf6a1679 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/f7faea9a44504e94b7cf8d54bf6a1679 2024-11-25T12:57:52,329 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/f7faea9a44504e94b7cf8d54bf6a1679, entries=150, sequenceid=360, filesize=12.0 K 2024-11-25T12:57:52,332 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/29814e698c324309832c12cacb140da4 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/29814e698c324309832c12cacb140da4 2024-11-25T12:57:52,358 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/29814e698c324309832c12cacb140da4, entries=150, sequenceid=360, filesize=12.0 K 2024-11-25T12:57:52,360 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/82d334f2dd654c87965902044d37535d as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/82d334f2dd654c87965902044d37535d 2024-11-25T12:57:52,367 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/82d334f2dd654c87965902044d37535d, entries=150, sequenceid=360, filesize=12.0 K 2024-11-25T12:57:52,369 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for e7fe900c90851bb165c5d4e631eedb85 in 359ms, sequenceid=360, compaction requested=true 2024-11-25T12:57:52,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e7fe900c90851bb165c5d4e631eedb85: 2024-11-25T12:57:52,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7fe900c90851bb165c5d4e631eedb85:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:57:52,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:57:52,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7fe900c90851bb165c5d4e631eedb85:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:57:52,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-25T12:57:52,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7fe900c90851bb165c5d4e631eedb85:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:57:52,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-25T12:57:52,370 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:57:52,370 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:57:52,372 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:57:52,372 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): e7fe900c90851bb165c5d4e631eedb85/C is initiating minor compaction (all files) 2024-11-25T12:57:52,372 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7fe900c90851bb165c5d4e631eedb85/C in TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:52,372 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/be2ddb98358c4bd5824fd85d13042779, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/560e3561e7704047a66074195052ac4b, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/67c297adf5d044e5bc377ef422495f8b, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/82d334f2dd654c87965902044d37535d] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp, totalSize=48.7 K 2024-11-25T12:57:52,373 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52326 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:57:52,373 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1541): e7fe900c90851bb165c5d4e631eedb85/A is initiating minor compaction (all files) 2024-11-25T12:57:52,373 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7fe900c90851bb165c5d4e631eedb85/A in TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:52,373 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/18bbda9e7c904728acb4211c2fe9baf9, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/03724ce6cff245bb86a911abb1492e5b, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/661d0293e834414280cde7fcd4660faa, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/f7faea9a44504e94b7cf8d54bf6a1679] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp, totalSize=51.1 K 2024-11-25T12:57:52,373 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting be2ddb98358c4bd5824fd85d13042779, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732539467116 2024-11-25T12:57:52,374 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18bbda9e7c904728acb4211c2fe9baf9, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732539467116 2024-11-25T12:57:52,374 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 560e3561e7704047a66074195052ac4b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732539469392 2024-11-25T12:57:52,374 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03724ce6cff245bb86a911abb1492e5b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732539469392 2024-11-25T12:57:52,374 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 661d0293e834414280cde7fcd4660faa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1732539470089 2024-11-25T12:57:52,375 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 67c297adf5d044e5bc377ef422495f8b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1732539470089 2024-11-25T12:57:52,375 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7faea9a44504e94b7cf8d54bf6a1679, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1732539471350 2024-11-25T12:57:52,375 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 82d334f2dd654c87965902044d37535d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1732539471350 2024-11-25T12:57:52,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on e7fe900c90851bb165c5d4e631eedb85 2024-11-25T12:57:52,404 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:57:52,404 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e7fe900c90851bb165c5d4e631eedb85 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-25T12:57:52,410 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7fe900c90851bb165c5d4e631eedb85#A#compaction#72 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:57:52,410 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/ce4433d1b9984f68a6d06824873efd03 is 50, key is test_row_0/A:col10/1732539471350/Put/seqid=0 2024-11-25T12:57:52,439 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7fe900c90851bb165c5d4e631eedb85#C#compaction#73 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:57:52,440 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/091b552b9d43410b97bb5b876c368cae is 50, key is test_row_0/C:col10/1732539471350/Put/seqid=0 2024-11-25T12:57:52,449 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/a49e7969a93e49dca4fd79c6c2e56e95 is 50, key is test_row_0/A:col10/1732539472398/Put/seqid=0 2024-11-25T12:57:52,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:52,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54984 deadline: 1732539532508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:52,526 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:52,526 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:52,526 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:52,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:52,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54986 deadline: 1732539532509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:52,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741909_1085 (size=13119) 2024-11-25T12:57:52,531 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:52,532 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:52,532 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:52,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:52,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55028 deadline: 1732539532517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:52,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:52,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55004 deadline: 1732539532524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:52,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:52,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:52,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:52,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:52,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:52,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:52,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741911_1087 (size=14741) 2024-11-25T12:57:52,541 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/a49e7969a93e49dca4fd79c6c2e56e95 2024-11-25T12:57:52,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741910_1086 (size=13119) 2024-11-25T12:57:52,571 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/ca77f667f3154a5d89e95e5f4bd52f95 is 50, key is test_row_0/B:col10/1732539472398/Put/seqid=0 2024-11-25T12:57:52,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741912_1088 (size=12301) 2024-11-25T12:57:52,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:52,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54984 deadline: 1732539532630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:52,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:52,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:52,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:52,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55028 deadline: 1732539532641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:52,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:52,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:52,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:52,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:52,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:52,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54986 deadline: 1732539532641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:52,643 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:52,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:52,643 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:52,643 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:52,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55004 deadline: 1732539532641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:52,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:52,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:52,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:52,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:52,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55028 deadline: 1732539532850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:52,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:52,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:52,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:52,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:52,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54984 deadline: 1732539532850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:52,857 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:52,857 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:52,857 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:52,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:52,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55004 deadline: 1732539532853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:52,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:52,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54986 deadline: 1732539532853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:52,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:52,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:52,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:52,859 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:52,859 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:52,859 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:52,937 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/ce4433d1b9984f68a6d06824873efd03 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/ce4433d1b9984f68a6d06824873efd03 2024-11-25T12:57:52,952 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e7fe900c90851bb165c5d4e631eedb85/A of e7fe900c90851bb165c5d4e631eedb85 into ce4433d1b9984f68a6d06824873efd03(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:57:52,953 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7fe900c90851bb165c5d4e631eedb85: 2024-11-25T12:57:52,953 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., storeName=e7fe900c90851bb165c5d4e631eedb85/A, priority=12, startTime=1732539472369; duration=0sec 2024-11-25T12:57:52,953 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:57:52,954 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7fe900c90851bb165c5d4e631eedb85:A 2024-11-25T12:57:52,954 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:57:52,954 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/091b552b9d43410b97bb5b876c368cae as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/091b552b9d43410b97bb5b876c368cae 2024-11-25T12:57:52,964 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e7fe900c90851bb165c5d4e631eedb85/C of e7fe900c90851bb165c5d4e631eedb85 into 091b552b9d43410b97bb5b876c368cae(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:57:52,964 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7fe900c90851bb165c5d4e631eedb85: 2024-11-25T12:57:52,964 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., storeName=e7fe900c90851bb165c5d4e631eedb85/C, priority=12, startTime=1732539472369; duration=0sec 2024-11-25T12:57:52,964 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:57:52,964 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7fe900c90851bb165c5d4e631eedb85:C 2024-11-25T12:57:52,965 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:57:52,965 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1541): e7fe900c90851bb165c5d4e631eedb85/B is initiating minor compaction (all files) 2024-11-25T12:57:52,965 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7fe900c90851bb165c5d4e631eedb85/B in TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:52,966 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/e1047cb31bc1480f946882e81c535024, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/8b84c83d2e234968bc17de037ff3e1c0, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/c06e1de741124d3cb7eac29e6066d4ed, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/29814e698c324309832c12cacb140da4] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp, totalSize=48.7 K 2024-11-25T12:57:52,966 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1047cb31bc1480f946882e81c535024, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732539467116 2024-11-25T12:57:52,967 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b84c83d2e234968bc17de037ff3e1c0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732539469392 2024-11-25T12:57:52,967 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting c06e1de741124d3cb7eac29e6066d4ed, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1732539470089 2024-11-25T12:57:52,969 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29814e698c324309832c12cacb140da4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1732539471350 2024-11-25T12:57:52,998 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7fe900c90851bb165c5d4e631eedb85#B#compaction#76 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:57:52,998 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/d267fcb8e9924be6bb04d4f83e6d9358 is 50, key is test_row_0/B:col10/1732539471350/Put/seqid=0 2024-11-25T12:57:53,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741913_1089 (size=13119) 2024-11-25T12:57:53,042 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/ca77f667f3154a5d89e95e5f4bd52f95 2024-11-25T12:57:53,052 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/d267fcb8e9924be6bb04d4f83e6d9358 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/d267fcb8e9924be6bb04d4f83e6d9358 2024-11-25T12:57:53,060 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e7fe900c90851bb165c5d4e631eedb85/B of e7fe900c90851bb165c5d4e631eedb85 into d267fcb8e9924be6bb04d4f83e6d9358(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:57:53,060 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7fe900c90851bb165c5d4e631eedb85: 2024-11-25T12:57:53,060 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., storeName=e7fe900c90851bb165c5d4e631eedb85/B, priority=12, startTime=1732539472369; duration=0sec 2024-11-25T12:57:53,062 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:57:53,062 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7fe900c90851bb165c5d4e631eedb85:B 2024-11-25T12:57:53,069 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/68ebe4850d12430aaafd4534eca24158 is 50, key is test_row_0/C:col10/1732539472398/Put/seqid=0 2024-11-25T12:57:53,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741914_1090 (size=12301) 2024-11-25T12:57:53,119 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/68ebe4850d12430aaafd4534eca24158 2024-11-25T12:57:53,126 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/a49e7969a93e49dca4fd79c6c2e56e95 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/a49e7969a93e49dca4fd79c6c2e56e95 2024-11-25T12:57:53,143 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/a49e7969a93e49dca4fd79c6c2e56e95, entries=200, sequenceid=371, filesize=14.4 K 2024-11-25T12:57:53,145 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/ca77f667f3154a5d89e95e5f4bd52f95 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/ca77f667f3154a5d89e95e5f4bd52f95 2024-11-25T12:57:53,160 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/ca77f667f3154a5d89e95e5f4bd52f95, entries=150, sequenceid=371, filesize=12.0 K 2024-11-25T12:57:53,162 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/68ebe4850d12430aaafd4534eca24158 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/68ebe4850d12430aaafd4534eca24158 2024-11-25T12:57:53,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:53,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55028 deadline: 1732539533160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:53,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:53,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:53,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:53,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:53,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54984 deadline: 1732539533171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:53,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:53,174 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:53,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55004 deadline: 1732539533171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:53,175 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:53,175 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:53,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:53,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:53,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54986 deadline: 1732539533171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:53,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:53,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:53,175 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:53,175 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:53,175 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:53,190 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/68ebe4850d12430aaafd4534eca24158, entries=150, sequenceid=371, filesize=12.0 K 2024-11-25T12:57:53,198 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-25T12:57:53,204 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for e7fe900c90851bb165c5d4e631eedb85 in 799ms, sequenceid=371, compaction requested=false 2024-11-25T12:57:53,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e7fe900c90851bb165c5d4e631eedb85: 2024-11-25T12:57:53,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-25T12:57:53,490 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:57:53,491 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:57:53,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-25T12:57:53,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-25T12:57:53,496 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:57:53,497 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:57:53,497 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:57:53,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-25T12:57:53,654 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-25T12:57:53,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:53,654 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing e7fe900c90851bb165c5d4e631eedb85 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-25T12:57:53,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/07c60b962fa54d9ea2e452b744c830af is 50, key is test_row_0/A:col10/1732539472504/Put/seqid=0 2024-11-25T12:57:53,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. as already flushing 2024-11-25T12:57:53,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on e7fe900c90851bb165c5d4e631eedb85 2024-11-25T12:57:53,719 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:53,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54984 deadline: 1732539533712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:53,720 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:53,720 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:53,720 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:53,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741915_1091 (size=12301) 2024-11-25T12:57:53,722 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/07c60b962fa54d9ea2e452b744c830af 2024-11-25T12:57:53,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:53,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55028 deadline: 1732539533720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:53,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:53,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:53,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:53,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55004 deadline: 1732539533721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:53,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:53,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:53,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:53,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:53,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:53,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54986 deadline: 1732539533721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:53,735 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:53,735 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:53,735 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:53,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/1a828ff91d9046febad9722f3f6b233f is 50, key is test_row_0/B:col10/1732539472504/Put/seqid=0 2024-11-25T12:57:53,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741916_1092 (size=12301) 2024-11-25T12:57:53,771 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/1a828ff91d9046febad9722f3f6b233f 2024-11-25T12:57:53,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-25T12:57:53,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/cf3f42ff66fd40799ceeef18606b7d05 is 50, key is test_row_0/C:col10/1732539472504/Put/seqid=0 2024-11-25T12:57:53,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741917_1093 (size=12301) 2024-11-25T12:57:53,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:53,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54984 deadline: 1732539533830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:53,832 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:53,832 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:53,832 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:53,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:53,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:53,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55004 deadline: 1732539533840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:53,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55028 deadline: 1732539533840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:53,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:53,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54986 deadline: 1732539533840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:53,845 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:53,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:53,846 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:53,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:53,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:53,846 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:53,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:53,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:53,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:54,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:54,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54984 deadline: 1732539534040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:54,043 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:54,043 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:54,043 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:54,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:54,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54986 deadline: 1732539534050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:54,051 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:54,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55004 deadline: 1732539534050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:54,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:54,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:54,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:54,053 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:54,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:54,053 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:54,053 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:54,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55028 deadline: 1732539534051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:54,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:54,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:54,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:54,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-25T12:57:54,229 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/cf3f42ff66fd40799ceeef18606b7d05 2024-11-25T12:57:54,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/07c60b962fa54d9ea2e452b744c830af as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/07c60b962fa54d9ea2e452b744c830af 2024-11-25T12:57:54,261 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/07c60b962fa54d9ea2e452b744c830af, entries=150, sequenceid=399, filesize=12.0 K 2024-11-25T12:57:54,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/1a828ff91d9046febad9722f3f6b233f as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/1a828ff91d9046febad9722f3f6b233f 2024-11-25T12:57:54,271 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/1a828ff91d9046febad9722f3f6b233f, entries=150, sequenceid=399, filesize=12.0 K 2024-11-25T12:57:54,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/cf3f42ff66fd40799ceeef18606b7d05 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/cf3f42ff66fd40799ceeef18606b7d05 2024-11-25T12:57:54,301 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/cf3f42ff66fd40799ceeef18606b7d05, entries=150, sequenceid=399, filesize=12.0 K 2024-11-25T12:57:54,302 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=67.09 KB/68700 for e7fe900c90851bb165c5d4e631eedb85 in 648ms, sequenceid=399, compaction requested=true 2024-11-25T12:57:54,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for e7fe900c90851bb165c5d4e631eedb85: 2024-11-25T12:57:54,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:54,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-25T12:57:54,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-25T12:57:54,307 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-25T12:57:54,307 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 808 msec 2024-11-25T12:57:54,309 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 817 msec 2024-11-25T12:57:54,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on e7fe900c90851bb165c5d4e631eedb85 2024-11-25T12:57:54,364 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:57:54,364 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e7fe900c90851bb165c5d4e631eedb85 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-25T12:57:54,376 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/1fa543b49e1442438b3c04793d1f69f3 is 50, key is test_row_0/A:col10/1732539474360/Put/seqid=0 2024-11-25T12:57:54,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741918_1094 (size=14741) 2024-11-25T12:57:54,392 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/1fa543b49e1442438b3c04793d1f69f3 2024-11-25T12:57:54,413 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/df4f69c320c94b7a85643f5a3c881015 is 50, key is test_row_0/B:col10/1732539474360/Put/seqid=0 2024-11-25T12:57:54,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741919_1095 (size=12301) 2024-11-25T12:57:54,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:54,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54984 deadline: 1732539534428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:54,460 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:54,460 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:54,460 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:54,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:54,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55028 deadline: 1732539534454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:54,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:54,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:54,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54986 deadline: 1732539534454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:54,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:54,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:54,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:54,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:54,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:54,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:54,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55004 deadline: 1732539534463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:54,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:54,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:54,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:54,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:54,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54984 deadline: 1732539534571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:54,573 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:54,573 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:54,573 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:54,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:54,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55028 deadline: 1732539534580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:54,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:54,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:54,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54986 deadline: 1732539534580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:54,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:54,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:54,586 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:54,586 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:54,587 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:54,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:54,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55004 deadline: 1732539534581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:54,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:54,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:54,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:54,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-25T12:57:54,631 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:57:54,636 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:57:54,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-25T12:57:54,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-25T12:57:54,644 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:57:54,645 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:57:54,645 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:57:54,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-25T12:57:54,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:54,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54984 deadline: 1732539534780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:54,782 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:54,782 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:54,782 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:54,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:54,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55028 deadline: 1732539534801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:54,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:54,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55004 deadline: 1732539534803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:54,805 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-25T12:57:54,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:54,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:54,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:54,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:54,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:54,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:54,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:54,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:54,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. as already flushing 2024-11-25T12:57:54,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54986 deadline: 1732539534803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:54,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:54,806 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:54,806 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:54,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:54,806 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:54,806 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:54,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:54,863 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/df4f69c320c94b7a85643f5a3c881015 2024-11-25T12:57:54,924 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/54ca4bc93867429abc43c7609fc046c1 is 50, key is test_row_0/C:col10/1732539474360/Put/seqid=0 2024-11-25T12:57:54,963 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-25T12:57:54,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:54,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. as already flushing 2024-11-25T12:57:54,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:54,968 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:54,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:54,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:54,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-25T12:57:54,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741920_1096 (size=12301) 2024-11-25T12:57:55,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:55,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54984 deadline: 1732539535092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:55,103 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:55,103 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:55,103 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:55,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:55,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55004 deadline: 1732539535115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:55,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:55,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54986 deadline: 1732539535115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:55,121 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:55,121 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:55,121 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:55,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:57:55,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55028 deadline: 1732539535116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:55,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:55,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:55,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:55,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:55,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:57:55,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:57:55,127 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-25T12:57:55,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:55,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. as already flushing 2024-11-25T12:57:55,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:55,130 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:55,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:55,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:55,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-25T12:57:55,290 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-25T12:57:55,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:55,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. as already flushing 2024-11-25T12:57:55,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:55,290 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:55,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:55,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:57:55,389 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/54ca4bc93867429abc43c7609fc046c1 2024-11-25T12:57:55,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/1fa543b49e1442438b3c04793d1f69f3 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/1fa543b49e1442438b3c04793d1f69f3 2024-11-25T12:57:55,405 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/1fa543b49e1442438b3c04793d1f69f3, entries=200, sequenceid=413, filesize=14.4 K 2024-11-25T12:57:55,408 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/df4f69c320c94b7a85643f5a3c881015 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/df4f69c320c94b7a85643f5a3c881015 2024-11-25T12:57:55,410 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-25T12:57:55,410 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:57:55,412 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-25T12:57:55,412 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:57:55,415 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-25T12:57:55,415 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:57:55,416 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:57:55,416 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:57:55,416 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:57:55,417 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:57:55,417 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:57:55,417 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:57:55,417 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:57:55,417 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:57:55,417 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:57:55,418 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/df4f69c320c94b7a85643f5a3c881015, entries=150, sequenceid=413, filesize=12.0 K 2024-11-25T12:57:55,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/54ca4bc93867429abc43c7609fc046c1 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/54ca4bc93867429abc43c7609fc046c1 2024-11-25T12:57:55,425 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-25T12:57:55,425 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:57:55,425 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:57:55,425 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:57:55,425 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:57:55,430 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/54ca4bc93867429abc43c7609fc046c1, entries=150, sequenceid=413, filesize=12.0 K 2024-11-25T12:57:55,431 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=140.89 KB/144270 for e7fe900c90851bb165c5d4e631eedb85 in 1066ms, sequenceid=413, compaction requested=true 2024-11-25T12:57:55,431 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e7fe900c90851bb165c5d4e631eedb85: 2024-11-25T12:57:55,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7fe900c90851bb165c5d4e631eedb85:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:57:55,431 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:57:55,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:57:55,431 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:57:55,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7fe900c90851bb165c5d4e631eedb85:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:57:55,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:57:55,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7fe900c90851bb165c5d4e631eedb85:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:57:55,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:57:55,433 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54902 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:57:55,433 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:57:55,433 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1541): e7fe900c90851bb165c5d4e631eedb85/A is initiating minor compaction (all files) 2024-11-25T12:57:55,433 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): e7fe900c90851bb165c5d4e631eedb85/B is initiating minor compaction (all files) 2024-11-25T12:57:55,433 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7fe900c90851bb165c5d4e631eedb85/A in TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:55,433 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7fe900c90851bb165c5d4e631eedb85/B in TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:55,433 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/ce4433d1b9984f68a6d06824873efd03, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/a49e7969a93e49dca4fd79c6c2e56e95, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/07c60b962fa54d9ea2e452b744c830af, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/1fa543b49e1442438b3c04793d1f69f3] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp, totalSize=53.6 K 2024-11-25T12:57:55,433 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/d267fcb8e9924be6bb04d4f83e6d9358, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/ca77f667f3154a5d89e95e5f4bd52f95, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/1a828ff91d9046febad9722f3f6b233f, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/df4f69c320c94b7a85643f5a3c881015] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp, totalSize=48.8 K 2024-11-25T12:57:55,433 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting d267fcb8e9924be6bb04d4f83e6d9358, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1732539471350 2024-11-25T12:57:55,434 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting ca77f667f3154a5d89e95e5f4bd52f95, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1732539472043 2024-11-25T12:57:55,434 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce4433d1b9984f68a6d06824873efd03, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1732539471350 2024-11-25T12:57:55,435 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a828ff91d9046febad9722f3f6b233f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1732539472504 2024-11-25T12:57:55,435 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting a49e7969a93e49dca4fd79c6c2e56e95, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1732539472043 2024-11-25T12:57:55,435 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07c60b962fa54d9ea2e452b744c830af, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1732539472504 2024-11-25T12:57:55,435 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting df4f69c320c94b7a85643f5a3c881015, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1732539473706 2024-11-25T12:57:55,435 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1fa543b49e1442438b3c04793d1f69f3, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1732539473706 2024-11-25T12:57:55,446 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7fe900c90851bb165c5d4e631eedb85#B#compaction#84 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:57:55,446 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-25T12:57:55,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:55,446 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/d61849da4ad04dc9975d6340d23fe654 is 50, key is test_row_0/B:col10/1732539474360/Put/seqid=0 2024-11-25T12:57:55,446 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing e7fe900c90851bb165c5d4e631eedb85 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-25T12:57:55,450 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7fe900c90851bb165c5d4e631eedb85#A#compaction#85 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:57:55,451 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/77bfbe9a30234a3dbbeba3b6529c0b61 is 50, key is test_row_0/A:col10/1732539474360/Put/seqid=0 2024-11-25T12:57:55,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/687b7c8de4ec4102bf87b5661700f9f0 is 50, key is test_row_0/A:col10/1732539474426/Put/seqid=0 2024-11-25T12:57:55,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741921_1097 (size=13255) 2024-11-25T12:57:55,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741922_1098 (size=13255) 2024-11-25T12:57:55,477 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/d61849da4ad04dc9975d6340d23fe654 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/d61849da4ad04dc9975d6340d23fe654 2024-11-25T12:57:55,484 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e7fe900c90851bb165c5d4e631eedb85/B of e7fe900c90851bb165c5d4e631eedb85 into d61849da4ad04dc9975d6340d23fe654(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:57:55,484 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7fe900c90851bb165c5d4e631eedb85: 2024-11-25T12:57:55,484 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., storeName=e7fe900c90851bb165c5d4e631eedb85/B, priority=12, startTime=1732539475431; duration=0sec 2024-11-25T12:57:55,484 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:57:55,485 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7fe900c90851bb165c5d4e631eedb85:B 2024-11-25T12:57:55,485 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:57:55,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741923_1099 (size=12301) 2024-11-25T12:57:55,487 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:57:55,487 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): e7fe900c90851bb165c5d4e631eedb85/C is initiating minor compaction (all files) 2024-11-25T12:57:55,487 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7fe900c90851bb165c5d4e631eedb85/C in TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:55,488 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/091b552b9d43410b97bb5b876c368cae, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/68ebe4850d12430aaafd4534eca24158, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/cf3f42ff66fd40799ceeef18606b7d05, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/54ca4bc93867429abc43c7609fc046c1] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp, totalSize=48.8 K 2024-11-25T12:57:55,488 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 091b552b9d43410b97bb5b876c368cae, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1732539471350 2024-11-25T12:57:55,488 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 68ebe4850d12430aaafd4534eca24158, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1732539472043 2024-11-25T12:57:55,489 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting cf3f42ff66fd40799ceeef18606b7d05, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1732539472504 2024-11-25T12:57:55,489 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 54ca4bc93867429abc43c7609fc046c1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1732539473706 2024-11-25T12:57:55,491 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=437 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/687b7c8de4ec4102bf87b5661700f9f0 2024-11-25T12:57:55,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/17f5ec6c5137471aaa235a4a46460e63 is 50, key is test_row_0/B:col10/1732539474426/Put/seqid=0 2024-11-25T12:57:55,512 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7fe900c90851bb165c5d4e631eedb85#C#compaction#88 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:57:55,513 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/973ec6bf486e41348a04d25074a1c1b6 is 50, key is test_row_0/C:col10/1732539474360/Put/seqid=0 2024-11-25T12:57:55,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741924_1100 (size=12301) 2024-11-25T12:57:55,528 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=437 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/17f5ec6c5137471aaa235a4a46460e63 2024-11-25T12:57:55,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741925_1101 (size=13255) 2024-11-25T12:57:55,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/2c7aecb0cc6c402cba83fb10e3c61b5e is 50, key is test_row_0/C:col10/1732539474426/Put/seqid=0 2024-11-25T12:57:55,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741926_1102 (size=12301) 2024-11-25T12:57:55,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on e7fe900c90851bb165c5d4e631eedb85 2024-11-25T12:57:55,613 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. as already flushing 2024-11-25T12:57:55,613 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-25T12:57:55,613 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:57:55,613 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:57:55,613 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:57:55,613 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:57:55,638 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-25T12:57:55,638 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:57:55,638 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:57:55,638 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:57:55,639 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-25T12:57:55,639 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:57:55,639 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:57:55,639 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:57:55,639 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:57:55,641 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-25T12:57:55,641 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:57:55,641 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:57:55,641 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:57:55,641 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:57:55,643 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:57:55,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-25T12:57:55,883 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/77bfbe9a30234a3dbbeba3b6529c0b61 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/77bfbe9a30234a3dbbeba3b6529c0b61 2024-11-25T12:57:55,891 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e7fe900c90851bb165c5d4e631eedb85/A of e7fe900c90851bb165c5d4e631eedb85 into 77bfbe9a30234a3dbbeba3b6529c0b61(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:57:55,891 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7fe900c90851bb165c5d4e631eedb85: 2024-11-25T12:57:55,891 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., storeName=e7fe900c90851bb165c5d4e631eedb85/A, priority=12, startTime=1732539475431; duration=0sec 2024-11-25T12:57:55,891 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:57:55,891 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7fe900c90851bb165c5d4e631eedb85:A 2024-11-25T12:57:55,952 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/973ec6bf486e41348a04d25074a1c1b6 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/973ec6bf486e41348a04d25074a1c1b6 2024-11-25T12:57:55,957 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=437 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/2c7aecb0cc6c402cba83fb10e3c61b5e 2024-11-25T12:57:55,959 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e7fe900c90851bb165c5d4e631eedb85/C of e7fe900c90851bb165c5d4e631eedb85 into 973ec6bf486e41348a04d25074a1c1b6(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:57:55,959 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7fe900c90851bb165c5d4e631eedb85: 2024-11-25T12:57:55,960 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., storeName=e7fe900c90851bb165c5d4e631eedb85/C, priority=12, startTime=1732539475431; duration=0sec 2024-11-25T12:57:55,960 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:57:55,960 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7fe900c90851bb165c5d4e631eedb85:C 2024-11-25T12:57:55,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/687b7c8de4ec4102bf87b5661700f9f0 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/687b7c8de4ec4102bf87b5661700f9f0 2024-11-25T12:57:55,972 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/687b7c8de4ec4102bf87b5661700f9f0, entries=150, sequenceid=437, filesize=12.0 K 2024-11-25T12:57:55,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/17f5ec6c5137471aaa235a4a46460e63 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/17f5ec6c5137471aaa235a4a46460e63 2024-11-25T12:57:55,981 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/17f5ec6c5137471aaa235a4a46460e63, entries=150, sequenceid=437, filesize=12.0 K 2024-11-25T12:57:55,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/2c7aecb0cc6c402cba83fb10e3c61b5e as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/2c7aecb0cc6c402cba83fb10e3c61b5e 2024-11-25T12:57:56,006 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/2c7aecb0cc6c402cba83fb10e3c61b5e, entries=150, sequenceid=437, filesize=12.0 K 2024-11-25T12:57:56,007 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=26.84 KB/27480 for e7fe900c90851bb165c5d4e631eedb85 in 561ms, sequenceid=437, compaction requested=false 2024-11-25T12:57:56,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for e7fe900c90851bb165c5d4e631eedb85: 2024-11-25T12:57:56,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:56,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-25T12:57:56,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-25T12:57:56,015 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-25T12:57:56,015 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3680 sec 2024-11-25T12:57:56,018 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.3800 sec 2024-11-25T12:57:56,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-25T12:57:56,801 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:57:57,355 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-25T12:57:59,571 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-25T12:57:59,571 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:57:59,571 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:57:59,572 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:57:59,572 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:57:59,572 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-25T12:57:59,572 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 85 2024-11-25T12:57:59,572 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 24 2024-11-25T12:57:59,572 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 81 2024-11-25T12:57:59,572 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 81 2024-11-25T12:57:59,572 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 83 2024-11-25T12:57:59,572 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-25T12:57:59,572 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1973 2024-11-25T12:57:59,572 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1958 2024-11-25T12:57:59,572 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-25T12:57:59,572 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 993 2024-11-25T12:57:59,572 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2978 rows 2024-11-25T12:57:59,572 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 973 2024-11-25T12:57:59,572 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2919 rows 2024-11-25T12:57:59,572 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T12:57:59,572 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T12:57:59,573 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:57:59,573 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:57:59,574 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:57:59,576 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-25T12:57:59,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-25T12:57:59,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-25T12:57:59,583 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732539479583"}]},"ts":"1732539479583"} 2024-11-25T12:57:59,585 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-25T12:57:59,585 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-25T12:57:59,587 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-25T12:57:59,592 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e7fe900c90851bb165c5d4e631eedb85, UNASSIGN}] 2024-11-25T12:57:59,593 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e7fe900c90851bb165c5d4e631eedb85, UNASSIGN 2024-11-25T12:57:59,595 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=e7fe900c90851bb165c5d4e631eedb85, regionState=CLOSING, regionLocation=1ff8009e463e,35905,1732539449246 2024-11-25T12:57:59,597 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e7fe900c90851bb165c5d4e631eedb85, UNASSIGN because future has completed 2024-11-25T12:57:59,599 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-25T12:57:59,599 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; CloseRegionProcedure e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246}] 2024-11-25T12:57:59,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-25T12:57:59,757 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(122): Close e7fe900c90851bb165c5d4e631eedb85 2024-11-25T12:57:59,757 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-25T12:57:59,758 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1722): Closing e7fe900c90851bb165c5d4e631eedb85, disabling compactions & flushes 2024-11-25T12:57:59,758 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:59,758 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:59,758 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. after waiting 0 ms 2024-11-25T12:57:59,758 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:57:59,759 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(2902): Flushing e7fe900c90851bb165c5d4e631eedb85 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-25T12:57:59,765 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/3401570f1aaa4547941af4b5f5e86279 is 50, key is test_row_1/A:col10/1732539475637/Put/seqid=0 2024-11-25T12:57:59,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741927_1103 (size=9857) 2024-11-25T12:57:59,776 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=448 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/3401570f1aaa4547941af4b5f5e86279 2024-11-25T12:57:59,798 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/1b1e1052ec434fd5b32582a9ed6ea144 is 50, key is test_row_1/B:col10/1732539475637/Put/seqid=0 2024-11-25T12:57:59,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741928_1104 (size=9857) 2024-11-25T12:57:59,829 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=448 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/1b1e1052ec434fd5b32582a9ed6ea144 2024-11-25T12:57:59,853 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/c0452ff832c24219bbc42e16ebfe65cd is 50, key is test_row_1/C:col10/1732539475637/Put/seqid=0 2024-11-25T12:57:59,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741929_1105 (size=9857) 2024-11-25T12:57:59,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-25T12:58:00,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-25T12:58:00,263 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=448 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/c0452ff832c24219bbc42e16ebfe65cd 2024-11-25T12:58:00,268 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/A/3401570f1aaa4547941af4b5f5e86279 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/3401570f1aaa4547941af4b5f5e86279 2024-11-25T12:58:00,273 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/3401570f1aaa4547941af4b5f5e86279, entries=100, sequenceid=448, filesize=9.6 K 2024-11-25T12:58:00,274 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/B/1b1e1052ec434fd5b32582a9ed6ea144 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/1b1e1052ec434fd5b32582a9ed6ea144 2024-11-25T12:58:00,280 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/1b1e1052ec434fd5b32582a9ed6ea144, entries=100, sequenceid=448, filesize=9.6 K 2024-11-25T12:58:00,281 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/.tmp/C/c0452ff832c24219bbc42e16ebfe65cd as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/c0452ff832c24219bbc42e16ebfe65cd 2024-11-25T12:58:00,287 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/c0452ff832c24219bbc42e16ebfe65cd, entries=100, sequenceid=448, filesize=9.6 K 2024-11-25T12:58:00,288 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for e7fe900c90851bb165c5d4e631eedb85 in 530ms, sequenceid=448, compaction requested=true 2024-11-25T12:58:00,288 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/a65a3e9e016b48769a3e1064776c6c4b, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/e91ac446bd494d89b700c4fd131a5168, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/af958271896a47769f71a2ec514c650e, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/949279ac08824960b0c6139042677c3a, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/0271da18e24741c8a5198ded16b64256, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/8acc07bcaeaf4fbcbbc9371ec17844c9, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/637eb22cbc2945888e7d4069c6c5f794, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/b2ac59dfdce744929a265975a24bd9fb, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/f8b1967e7ead47f4a3293498476e527f, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/8176aa47487a4eaea3ceb972aa3dce44, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/880aa15661fd4635988f299f7742698f, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/87e70f098c944995afe1d7c489f002bc, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/4df90064a0c94f25b820c9e377429e2f, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/5d386441780d4718bb4660845cd830e8, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/1e054310565b4cc39c7e4ac67e31422a, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/e487bcf33fa44e7d91432d6c5319f2c9, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/791057c8351941b6a807b408f2a0d253, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/12e4a0449da246319b5e69c735feab2d, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/261e9ca73b73459eb1fdbb8f8a43efce, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/18bbda9e7c904728acb4211c2fe9baf9, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/ec9a7dbf37214cfd9d5e3403864eff5d, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/03724ce6cff245bb86a911abb1492e5b, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/661d0293e834414280cde7fcd4660faa, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/ce4433d1b9984f68a6d06824873efd03, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/f7faea9a44504e94b7cf8d54bf6a1679, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/a49e7969a93e49dca4fd79c6c2e56e95, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/07c60b962fa54d9ea2e452b744c830af, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/1fa543b49e1442438b3c04793d1f69f3] to archive 2024-11-25T12:58:00,291 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T12:58:00,297 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/a65a3e9e016b48769a3e1064776c6c4b to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/a65a3e9e016b48769a3e1064776c6c4b 2024-11-25T12:58:00,299 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/e91ac446bd494d89b700c4fd131a5168 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/e91ac446bd494d89b700c4fd131a5168 2024-11-25T12:58:00,301 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/af958271896a47769f71a2ec514c650e to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/af958271896a47769f71a2ec514c650e 2024-11-25T12:58:00,303 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/949279ac08824960b0c6139042677c3a to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/949279ac08824960b0c6139042677c3a 2024-11-25T12:58:00,306 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/0271da18e24741c8a5198ded16b64256 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/0271da18e24741c8a5198ded16b64256 2024-11-25T12:58:00,308 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/8acc07bcaeaf4fbcbbc9371ec17844c9 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/8acc07bcaeaf4fbcbbc9371ec17844c9 2024-11-25T12:58:00,310 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/637eb22cbc2945888e7d4069c6c5f794 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/637eb22cbc2945888e7d4069c6c5f794 2024-11-25T12:58:00,311 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/b2ac59dfdce744929a265975a24bd9fb to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/b2ac59dfdce744929a265975a24bd9fb 2024-11-25T12:58:00,314 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/f8b1967e7ead47f4a3293498476e527f to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/f8b1967e7ead47f4a3293498476e527f 2024-11-25T12:58:00,315 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/8176aa47487a4eaea3ceb972aa3dce44 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/8176aa47487a4eaea3ceb972aa3dce44 2024-11-25T12:58:00,316 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/880aa15661fd4635988f299f7742698f to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/880aa15661fd4635988f299f7742698f 2024-11-25T12:58:00,318 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/87e70f098c944995afe1d7c489f002bc to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/87e70f098c944995afe1d7c489f002bc 2024-11-25T12:58:00,319 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/4df90064a0c94f25b820c9e377429e2f to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/4df90064a0c94f25b820c9e377429e2f 2024-11-25T12:58:00,320 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/5d386441780d4718bb4660845cd830e8 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/5d386441780d4718bb4660845cd830e8 2024-11-25T12:58:00,321 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/1e054310565b4cc39c7e4ac67e31422a to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/1e054310565b4cc39c7e4ac67e31422a 2024-11-25T12:58:00,323 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/e487bcf33fa44e7d91432d6c5319f2c9 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/e487bcf33fa44e7d91432d6c5319f2c9 2024-11-25T12:58:00,324 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/791057c8351941b6a807b408f2a0d253 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/791057c8351941b6a807b408f2a0d253 2024-11-25T12:58:00,325 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/12e4a0449da246319b5e69c735feab2d to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/12e4a0449da246319b5e69c735feab2d 2024-11-25T12:58:00,327 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/261e9ca73b73459eb1fdbb8f8a43efce to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/261e9ca73b73459eb1fdbb8f8a43efce 2024-11-25T12:58:00,328 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/18bbda9e7c904728acb4211c2fe9baf9 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/18bbda9e7c904728acb4211c2fe9baf9 2024-11-25T12:58:00,330 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/ec9a7dbf37214cfd9d5e3403864eff5d to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/ec9a7dbf37214cfd9d5e3403864eff5d 2024-11-25T12:58:00,333 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/03724ce6cff245bb86a911abb1492e5b to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/03724ce6cff245bb86a911abb1492e5b 2024-11-25T12:58:00,335 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/661d0293e834414280cde7fcd4660faa to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/661d0293e834414280cde7fcd4660faa 2024-11-25T12:58:00,336 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/ce4433d1b9984f68a6d06824873efd03 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/ce4433d1b9984f68a6d06824873efd03 2024-11-25T12:58:00,337 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/f7faea9a44504e94b7cf8d54bf6a1679 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/f7faea9a44504e94b7cf8d54bf6a1679 2024-11-25T12:58:00,339 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/a49e7969a93e49dca4fd79c6c2e56e95 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/a49e7969a93e49dca4fd79c6c2e56e95 2024-11-25T12:58:00,345 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/07c60b962fa54d9ea2e452b744c830af to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/07c60b962fa54d9ea2e452b744c830af 2024-11-25T12:58:00,352 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/1fa543b49e1442438b3c04793d1f69f3 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/1fa543b49e1442438b3c04793d1f69f3 2024-11-25T12:58:00,366 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/575539a855ad4ac0a7f1840d0476f6a1, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/09625318e20042b0a9f46bbc51858d8f, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/00b5380c8cf841e2a19380578a94339d, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/fbf6d8c94a2f46e5bb1acc1bcae65984, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/7458dfc0991a4c908e5e6bcc7ef9f353, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/0f12ca889b7342f2bd8855de5203c166, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/363763aca5cc4c48b5348ba8837f2b1a, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/8744df911fba423397c05675909c341f, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/2d0b71414d26400a965aed4d7d54205f, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/b7009c14f6814fd9acece2ad30bf0e99, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/fe50a6680a48498cbe4c91f033a0dc3f, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/030a9fe4f794449da3ba4f0d83c8f7ee, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/4c0473c5424f46a99eb53620d2a31bda, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/798a4005e9d045c4a16828210990a32e, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/28cc338e22254dd8948534e4463c0953, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/57e656f92c2b44b3bcf31b011141313a, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/37f6f3364c5f413d9024da547a427fdb, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/fa73283293ac4ecbbe3f1b2f7d344f59, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/6defa8c81c2d4daab663ed5b4247cfc9, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/e1047cb31bc1480f946882e81c535024, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/8cbfe687d6b1411e89b5f01a7910d877, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/8b84c83d2e234968bc17de037ff3e1c0, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/c06e1de741124d3cb7eac29e6066d4ed, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/d267fcb8e9924be6bb04d4f83e6d9358, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/29814e698c324309832c12cacb140da4, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/ca77f667f3154a5d89e95e5f4bd52f95, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/1a828ff91d9046febad9722f3f6b233f, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/df4f69c320c94b7a85643f5a3c881015] to archive 2024-11-25T12:58:00,367 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T12:58:00,369 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/575539a855ad4ac0a7f1840d0476f6a1 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/575539a855ad4ac0a7f1840d0476f6a1 2024-11-25T12:58:00,370 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/09625318e20042b0a9f46bbc51858d8f to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/09625318e20042b0a9f46bbc51858d8f 2024-11-25T12:58:00,371 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/00b5380c8cf841e2a19380578a94339d to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/00b5380c8cf841e2a19380578a94339d 2024-11-25T12:58:00,372 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/fbf6d8c94a2f46e5bb1acc1bcae65984 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/fbf6d8c94a2f46e5bb1acc1bcae65984 2024-11-25T12:58:00,374 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/7458dfc0991a4c908e5e6bcc7ef9f353 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/7458dfc0991a4c908e5e6bcc7ef9f353 2024-11-25T12:58:00,375 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/0f12ca889b7342f2bd8855de5203c166 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/0f12ca889b7342f2bd8855de5203c166 2024-11-25T12:58:00,376 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/363763aca5cc4c48b5348ba8837f2b1a to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/363763aca5cc4c48b5348ba8837f2b1a 2024-11-25T12:58:00,378 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/8744df911fba423397c05675909c341f to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/8744df911fba423397c05675909c341f 2024-11-25T12:58:00,382 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/2d0b71414d26400a965aed4d7d54205f to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/2d0b71414d26400a965aed4d7d54205f 2024-11-25T12:58:00,386 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/b7009c14f6814fd9acece2ad30bf0e99 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/b7009c14f6814fd9acece2ad30bf0e99 2024-11-25T12:58:00,389 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/fe50a6680a48498cbe4c91f033a0dc3f to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/fe50a6680a48498cbe4c91f033a0dc3f 2024-11-25T12:58:00,391 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/030a9fe4f794449da3ba4f0d83c8f7ee to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/030a9fe4f794449da3ba4f0d83c8f7ee 2024-11-25T12:58:00,393 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/4c0473c5424f46a99eb53620d2a31bda to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/4c0473c5424f46a99eb53620d2a31bda 2024-11-25T12:58:00,394 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/798a4005e9d045c4a16828210990a32e to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/798a4005e9d045c4a16828210990a32e 2024-11-25T12:58:00,396 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/28cc338e22254dd8948534e4463c0953 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/28cc338e22254dd8948534e4463c0953 2024-11-25T12:58:00,397 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/57e656f92c2b44b3bcf31b011141313a to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/57e656f92c2b44b3bcf31b011141313a 2024-11-25T12:58:00,399 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/37f6f3364c5f413d9024da547a427fdb to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/37f6f3364c5f413d9024da547a427fdb 2024-11-25T12:58:00,400 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/fa73283293ac4ecbbe3f1b2f7d344f59 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/fa73283293ac4ecbbe3f1b2f7d344f59 2024-11-25T12:58:00,407 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/6defa8c81c2d4daab663ed5b4247cfc9 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/6defa8c81c2d4daab663ed5b4247cfc9 2024-11-25T12:58:00,409 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/e1047cb31bc1480f946882e81c535024 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/e1047cb31bc1480f946882e81c535024 2024-11-25T12:58:00,411 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/8cbfe687d6b1411e89b5f01a7910d877 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/8cbfe687d6b1411e89b5f01a7910d877 2024-11-25T12:58:00,413 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/8b84c83d2e234968bc17de037ff3e1c0 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/8b84c83d2e234968bc17de037ff3e1c0 2024-11-25T12:58:00,414 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/c06e1de741124d3cb7eac29e6066d4ed to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/c06e1de741124d3cb7eac29e6066d4ed 2024-11-25T12:58:00,416 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/d267fcb8e9924be6bb04d4f83e6d9358 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/d267fcb8e9924be6bb04d4f83e6d9358 2024-11-25T12:58:00,418 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/29814e698c324309832c12cacb140da4 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/29814e698c324309832c12cacb140da4 2024-11-25T12:58:00,419 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/ca77f667f3154a5d89e95e5f4bd52f95 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/ca77f667f3154a5d89e95e5f4bd52f95 2024-11-25T12:58:00,421 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/1a828ff91d9046febad9722f3f6b233f to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/1a828ff91d9046febad9722f3f6b233f 2024-11-25T12:58:00,425 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/df4f69c320c94b7a85643f5a3c881015 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/df4f69c320c94b7a85643f5a3c881015 2024-11-25T12:58:00,428 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/77d7dfae1582430b957c9c92c665a62c, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/cf30b6d70cef4015991561c23700f8b7, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/b04f42b3a5f9431d908a0311dbf03b07, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/fc7c6b3e700043ef8fac8f8c72e63e9e, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/c6367c2ee92a475ea4751cc9ae7efc01, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/873bb666be514c04ad8b935de5bbbccd, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/3fec7ba60a634b419778d9ae2c79cb51, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/eb167a54f33d4a0e80ffa73416a1509d, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/48d34ab99b3741eb941bb6d28abeb522, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/26ccdd28c269477886a50316129ed767, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/690288bed1114f7c90f112241cf546e8, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/20821f68503a4213b8be5890d203bcde, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/197d05894e1b4f19996bda24406dae7f, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/f28a6a68d93a4e9580e0a282070e1c78, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/fb05acba06d64a4696e0e59c3182b872, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/550f6c82dcd341428966a9c43fd1e1f5, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/8c3daa5bdbda48b0b247bcbee49493e8, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/73923858f942427f8f83639ab50ab7d7, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/29df4b2f6c7741538af70d5bf16a7fba, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/be2ddb98358c4bd5824fd85d13042779, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/50bf4f0c88b448f18d515cde53083631, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/560e3561e7704047a66074195052ac4b, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/67c297adf5d044e5bc377ef422495f8b, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/091b552b9d43410b97bb5b876c368cae, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/82d334f2dd654c87965902044d37535d, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/68ebe4850d12430aaafd4534eca24158, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/cf3f42ff66fd40799ceeef18606b7d05, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/54ca4bc93867429abc43c7609fc046c1] to archive 2024-11-25T12:58:00,429 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T12:58:00,432 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/77d7dfae1582430b957c9c92c665a62c to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/77d7dfae1582430b957c9c92c665a62c 2024-11-25T12:58:00,433 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/cf30b6d70cef4015991561c23700f8b7 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/cf30b6d70cef4015991561c23700f8b7 2024-11-25T12:58:00,435 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/b04f42b3a5f9431d908a0311dbf03b07 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/b04f42b3a5f9431d908a0311dbf03b07 2024-11-25T12:58:00,436 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/fc7c6b3e700043ef8fac8f8c72e63e9e to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/fc7c6b3e700043ef8fac8f8c72e63e9e 2024-11-25T12:58:00,438 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/c6367c2ee92a475ea4751cc9ae7efc01 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/c6367c2ee92a475ea4751cc9ae7efc01 2024-11-25T12:58:00,440 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/873bb666be514c04ad8b935de5bbbccd to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/873bb666be514c04ad8b935de5bbbccd 2024-11-25T12:58:00,441 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/3fec7ba60a634b419778d9ae2c79cb51 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/3fec7ba60a634b419778d9ae2c79cb51 2024-11-25T12:58:00,443 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/eb167a54f33d4a0e80ffa73416a1509d to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/eb167a54f33d4a0e80ffa73416a1509d 2024-11-25T12:58:00,445 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/48d34ab99b3741eb941bb6d28abeb522 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/48d34ab99b3741eb941bb6d28abeb522 2024-11-25T12:58:00,447 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/26ccdd28c269477886a50316129ed767 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/26ccdd28c269477886a50316129ed767 2024-11-25T12:58:00,448 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/690288bed1114f7c90f112241cf546e8 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/690288bed1114f7c90f112241cf546e8 2024-11-25T12:58:00,450 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/20821f68503a4213b8be5890d203bcde to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/20821f68503a4213b8be5890d203bcde 2024-11-25T12:58:00,452 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/197d05894e1b4f19996bda24406dae7f to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/197d05894e1b4f19996bda24406dae7f 2024-11-25T12:58:00,454 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/f28a6a68d93a4e9580e0a282070e1c78 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/f28a6a68d93a4e9580e0a282070e1c78 2024-11-25T12:58:00,456 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/fb05acba06d64a4696e0e59c3182b872 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/fb05acba06d64a4696e0e59c3182b872 2024-11-25T12:58:00,458 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/550f6c82dcd341428966a9c43fd1e1f5 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/550f6c82dcd341428966a9c43fd1e1f5 2024-11-25T12:58:00,460 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/8c3daa5bdbda48b0b247bcbee49493e8 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/8c3daa5bdbda48b0b247bcbee49493e8 2024-11-25T12:58:00,461 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/73923858f942427f8f83639ab50ab7d7 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/73923858f942427f8f83639ab50ab7d7 2024-11-25T12:58:00,463 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/29df4b2f6c7741538af70d5bf16a7fba to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/29df4b2f6c7741538af70d5bf16a7fba 2024-11-25T12:58:00,464 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/be2ddb98358c4bd5824fd85d13042779 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/be2ddb98358c4bd5824fd85d13042779 2024-11-25T12:58:00,466 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/50bf4f0c88b448f18d515cde53083631 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/50bf4f0c88b448f18d515cde53083631 2024-11-25T12:58:00,467 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/560e3561e7704047a66074195052ac4b to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/560e3561e7704047a66074195052ac4b 2024-11-25T12:58:00,469 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/67c297adf5d044e5bc377ef422495f8b to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/67c297adf5d044e5bc377ef422495f8b 2024-11-25T12:58:00,470 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/091b552b9d43410b97bb5b876c368cae to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/091b552b9d43410b97bb5b876c368cae 2024-11-25T12:58:00,471 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/82d334f2dd654c87965902044d37535d to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/82d334f2dd654c87965902044d37535d 2024-11-25T12:58:00,472 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/68ebe4850d12430aaafd4534eca24158 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/68ebe4850d12430aaafd4534eca24158 2024-11-25T12:58:00,474 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/cf3f42ff66fd40799ceeef18606b7d05 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/cf3f42ff66fd40799ceeef18606b7d05 2024-11-25T12:58:00,475 DEBUG [StoreCloser-TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/54ca4bc93867429abc43c7609fc046c1 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/54ca4bc93867429abc43c7609fc046c1 2024-11-25T12:58:00,484 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/recovered.edits/451.seqid, newMaxSeqId=451, maxSeqId=1 2024-11-25T12:58:00,485 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. 2024-11-25T12:58:00,485 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1676): Region close journal for e7fe900c90851bb165c5d4e631eedb85: Waiting for close lock at 1732539479758Running coprocessor pre-close hooks at 1732539479758Disabling compacts and flushes for region at 1732539479758Disabling writes for close at 1732539479758Obtaining lock to block concurrent updates at 1732539479759 (+1 ms)Preparing flush snapshotting stores in e7fe900c90851bb165c5d4e631eedb85 at 1732539479759Finished memstore snapshotting TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1732539479759Flushing stores of TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85. at 1732539479760 (+1 ms)Flushing e7fe900c90851bb165c5d4e631eedb85/A: creating writer at 1732539479760Flushing e7fe900c90851bb165c5d4e631eedb85/A: appending metadata at 1732539479764 (+4 ms)Flushing e7fe900c90851bb165c5d4e631eedb85/A: closing flushed file at 1732539479764Flushing e7fe900c90851bb165c5d4e631eedb85/B: creating writer at 1732539479791 (+27 ms)Flushing e7fe900c90851bb165c5d4e631eedb85/B: appending metadata at 1732539479797 (+6 ms)Flushing e7fe900c90851bb165c5d4e631eedb85/B: closing flushed file at 1732539479798 (+1 ms)Flushing e7fe900c90851bb165c5d4e631eedb85/C: creating writer at 1732539479843 (+45 ms)Flushing e7fe900c90851bb165c5d4e631eedb85/C: appending metadata at 1732539479852 (+9 ms)Flushing e7fe900c90851bb165c5d4e631eedb85/C: closing flushed file at 1732539479852Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@31a56c91: reopening flushed file at 1732539480267 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@577efa76: reopening flushed file at 1732539480273 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7ff56d2c: reopening flushed file at 1732539480280 (+7 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for e7fe900c90851bb165c5d4e631eedb85 in 530ms, sequenceid=448, compaction requested=true at 1732539480288 (+8 ms)Writing region close event to WAL at 1732539480477 (+189 ms)Running coprocessor post-close hooks at 1732539480484 (+7 ms)Closed at 1732539480485 (+1 ms) 2024-11-25T12:58:00,487 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(157): Closed e7fe900c90851bb165c5d4e631eedb85 2024-11-25T12:58:00,488 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=e7fe900c90851bb165c5d4e631eedb85, regionState=CLOSED 2024-11-25T12:58:00,489 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE, hasLock=false; CloseRegionProcedure e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 because future has completed 2024-11-25T12:58:00,491 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-25T12:58:00,492 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseRegionProcedure e7fe900c90851bb165c5d4e631eedb85, server=1ff8009e463e,35905,1732539449246 in 891 msec 2024-11-25T12:58:00,494 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=25, resume processing ppid=24 2024-11-25T12:58:00,494 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=24, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e7fe900c90851bb165c5d4e631eedb85, UNASSIGN in 900 msec 2024-11-25T12:58:00,496 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-25T12:58:00,496 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 907 msec 2024-11-25T12:58:00,497 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732539480497"}]},"ts":"1732539480497"} 2024-11-25T12:58:00,498 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-25T12:58:00,498 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-25T12:58:00,500 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 922 msec 2024-11-25T12:58:00,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-25T12:58:00,724 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:58:00,731 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-25T12:58:00,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T12:58:00,741 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=27, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T12:58:00,743 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=27, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T12:58:00,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-25T12:58:00,767 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85 2024-11-25T12:58:00,820 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A, FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B, FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C, FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/recovered.edits] 2024-11-25T12:58:00,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-25T12:58:00,853 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/3401570f1aaa4547941af4b5f5e86279 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/3401570f1aaa4547941af4b5f5e86279 2024-11-25T12:58:00,855 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/687b7c8de4ec4102bf87b5661700f9f0 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/687b7c8de4ec4102bf87b5661700f9f0 2024-11-25T12:58:00,857 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/77bfbe9a30234a3dbbeba3b6529c0b61 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/A/77bfbe9a30234a3dbbeba3b6529c0b61 2024-11-25T12:58:00,860 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/17f5ec6c5137471aaa235a4a46460e63 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/17f5ec6c5137471aaa235a4a46460e63 2024-11-25T12:58:00,862 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/1b1e1052ec434fd5b32582a9ed6ea144 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/1b1e1052ec434fd5b32582a9ed6ea144 2024-11-25T12:58:00,863 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/d61849da4ad04dc9975d6340d23fe654 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/B/d61849da4ad04dc9975d6340d23fe654 2024-11-25T12:58:00,867 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/2c7aecb0cc6c402cba83fb10e3c61b5e to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/2c7aecb0cc6c402cba83fb10e3c61b5e 2024-11-25T12:58:00,868 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/973ec6bf486e41348a04d25074a1c1b6 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/973ec6bf486e41348a04d25074a1c1b6 2024-11-25T12:58:00,870 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/c0452ff832c24219bbc42e16ebfe65cd to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/C/c0452ff832c24219bbc42e16ebfe65cd 2024-11-25T12:58:00,874 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/recovered.edits/451.seqid to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85/recovered.edits/451.seqid 2024-11-25T12:58:00,875 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/e7fe900c90851bb165c5d4e631eedb85 2024-11-25T12:58:00,875 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-25T12:58:00,883 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=27, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T12:58:00,888 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-25T12:58:00,895 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-25T12:58:00,951 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-25T12:58:00,958 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=27, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T12:58:00,958 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-25T12:58:00,959 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732539480958"}]},"ts":"9223372036854775807"} 2024-11-25T12:58:00,964 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-25T12:58:00,964 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => e7fe900c90851bb165c5d4e631eedb85, NAME => 'TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85.', STARTKEY => '', ENDKEY => ''}] 2024-11-25T12:58:00,965 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-25T12:58:00,966 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732539480965"}]},"ts":"9223372036854775807"} 2024-11-25T12:58:00,971 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-25T12:58:00,976 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=27, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T12:58:00,979 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 244 msec 2024-11-25T12:58:01,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-25T12:58:01,072 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-25T12:58:01,072 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:58:01,088 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=237 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-348484797_22 at /127.0.0.1:59140 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;1ff8009e463e:35905-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=451 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=909 (was 701) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1451 (was 2036) 2024-11-25T12:58:01,100 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=237, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=909, ProcessCount=11, AvailableMemoryMB=1451 2024-11-25T12:58:01,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-25T12:58:01,102 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T12:58:01,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] procedure2.ProcedureExecutor(1139): Stored pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-25T12:58:01,106 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T12:58:01,106 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:01,106 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 28 2024-11-25T12:58:01,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-25T12:58:01,109 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T12:58:01,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741930_1106 (size=959) 2024-11-25T12:58:01,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-25T12:58:01,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-25T12:58:01,538 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a 2024-11-25T12:58:01,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741931_1107 (size=53) 2024-11-25T12:58:01,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-25T12:58:01,947 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:58:01,948 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 9806b0fa8f18a18b9b918dba8c42bc67, disabling compactions & flushes 2024-11-25T12:58:01,948 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:01,948 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:01,948 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. after waiting 0 ms 2024-11-25T12:58:01,948 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:01,948 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:01,948 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 9806b0fa8f18a18b9b918dba8c42bc67: Waiting for close lock at 1732539481947Disabling compacts and flushes for region at 1732539481947Disabling writes for close at 1732539481948 (+1 ms)Writing region close event to WAL at 1732539481948Closed at 1732539481948 2024-11-25T12:58:01,953 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T12:58:01,953 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732539481953"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732539481953"}]},"ts":"1732539481953"} 2024-11-25T12:58:01,961 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T12:58:01,962 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T12:58:01,962 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732539481962"}]},"ts":"1732539481962"} 2024-11-25T12:58:01,965 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-25T12:58:01,966 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9806b0fa8f18a18b9b918dba8c42bc67, ASSIGN}] 2024-11-25T12:58:01,968 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9806b0fa8f18a18b9b918dba8c42bc67, ASSIGN 2024-11-25T12:58:01,970 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9806b0fa8f18a18b9b918dba8c42bc67, ASSIGN; state=OFFLINE, location=1ff8009e463e,35905,1732539449246; forceNewPlan=false, retain=false 2024-11-25T12:58:02,120 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=9806b0fa8f18a18b9b918dba8c42bc67, regionState=OPENING, regionLocation=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:02,122 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9806b0fa8f18a18b9b918dba8c42bc67, ASSIGN because future has completed 2024-11-25T12:58:02,123 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246}] 2024-11-25T12:58:02,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-25T12:58:02,278 INFO [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:02,279 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} 2024-11-25T12:58:02,279 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:02,279 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:58:02,279 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for 9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:02,280 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for 9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:02,283 INFO [StoreOpener-9806b0fa8f18a18b9b918dba8c42bc67-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:02,285 INFO [StoreOpener-9806b0fa8f18a18b9b918dba8c42bc67-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9806b0fa8f18a18b9b918dba8c42bc67 columnFamilyName A 2024-11-25T12:58:02,285 DEBUG [StoreOpener-9806b0fa8f18a18b9b918dba8c42bc67-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:02,286 INFO [StoreOpener-9806b0fa8f18a18b9b918dba8c42bc67-1 {}] regionserver.HStore(327): Store=9806b0fa8f18a18b9b918dba8c42bc67/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:58:02,286 INFO [StoreOpener-9806b0fa8f18a18b9b918dba8c42bc67-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:02,287 INFO [StoreOpener-9806b0fa8f18a18b9b918dba8c42bc67-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9806b0fa8f18a18b9b918dba8c42bc67 columnFamilyName B 2024-11-25T12:58:02,287 DEBUG [StoreOpener-9806b0fa8f18a18b9b918dba8c42bc67-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:02,288 INFO [StoreOpener-9806b0fa8f18a18b9b918dba8c42bc67-1 {}] regionserver.HStore(327): Store=9806b0fa8f18a18b9b918dba8c42bc67/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:58:02,288 INFO [StoreOpener-9806b0fa8f18a18b9b918dba8c42bc67-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:02,289 INFO [StoreOpener-9806b0fa8f18a18b9b918dba8c42bc67-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9806b0fa8f18a18b9b918dba8c42bc67 columnFamilyName C 2024-11-25T12:58:02,289 DEBUG [StoreOpener-9806b0fa8f18a18b9b918dba8c42bc67-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:02,289 INFO [StoreOpener-9806b0fa8f18a18b9b918dba8c42bc67-1 {}] regionserver.HStore(327): Store=9806b0fa8f18a18b9b918dba8c42bc67/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:58:02,289 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for 9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:02,290 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:02,290 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:02,291 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for 9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:02,291 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for 9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:02,292 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-25T12:58:02,293 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for 9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:02,299 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T12:58:02,300 INFO [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1114): Opened 9806b0fa8f18a18b9b918dba8c42bc67; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73056350, jitterRate=0.08862444758415222}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-25T12:58:02,300 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:02,303 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for 9806b0fa8f18a18b9b918dba8c42bc67: Running coprocessor pre-open hook at 1732539482280Writing region info on filesystem at 1732539482280Initializing all the Stores at 1732539482281 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732539482281Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732539482282 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732539482283 (+1 ms)Cleaning up temporary data from old regions at 1732539482291 (+8 ms)Running coprocessor post-open hooks at 1732539482300 (+9 ms)Region opened successfully at 1732539482303 (+3 ms) 2024-11-25T12:58:02,304 INFO [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., pid=30, masterSystemTime=1732539482275 2024-11-25T12:58:02,306 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:02,307 INFO [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:02,307 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=9806b0fa8f18a18b9b918dba8c42bc67, regionState=OPEN, openSeqNum=2, regionLocation=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:02,308 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46289 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=1ff8009e463e,35905,1732539449246, table=TestAcidGuarantees, region=9806b0fa8f18a18b9b918dba8c42bc67. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-25T12:58:02,308 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 because future has completed 2024-11-25T12:58:02,311 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-25T12:58:02,311 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 in 186 msec 2024-11-25T12:58:02,313 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-25T12:58:02,313 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9806b0fa8f18a18b9b918dba8c42bc67, ASSIGN in 345 msec 2024-11-25T12:58:02,314 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T12:58:02,314 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732539482314"}]},"ts":"1732539482314"} 2024-11-25T12:58:02,316 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-25T12:58:02,317 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T12:58:02,319 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2150 sec 2024-11-25T12:58:03,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-25T12:58:03,262 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:58:03,263 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b4db6c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:58:03,263 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ff8009e463e,46289,-1 for getting cluster id 2024-11-25T12:58:03,263 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:58:03,268 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb972fd5-cdcb-422b-acfe-41fc80d301d9' 2024-11-25T12:58:03,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:58:03,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb972fd5-cdcb-422b-acfe-41fc80d301d9" 2024-11-25T12:58:03,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59fc840e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:58:03,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ff8009e463e,46289,-1] 2024-11-25T12:58:03,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:58:03,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:58:03,270 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44682, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:58:03,271 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2df92b02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:58:03,272 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:58:03,273 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,35905,1732539449246, seqNum=-1] 2024-11-25T12:58:03,273 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:58:03,276 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42454, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:58:03,278 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-25T12:58:03,279 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 1ff8009e463e,46289,1732539448369 2024-11-25T12:58:03,279 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@34f193ca 2024-11-25T12:58:03,279 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-25T12:58:03,281 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44696, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-25T12:58:03,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-25T12:58:03,289 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T12:58:03,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-25T12:58:03,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741932_1108 (size=995) 2024-11-25T12:58:03,347 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-25T12:58:03,347 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-25T12:58:03,351 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-25T12:58:03,358 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9806b0fa8f18a18b9b918dba8c42bc67, REOPEN/MOVE}] 2024-11-25T12:58:03,359 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9806b0fa8f18a18b9b918dba8c42bc67, REOPEN/MOVE 2024-11-25T12:58:03,360 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=9806b0fa8f18a18b9b918dba8c42bc67, regionState=CLOSING, regionLocation=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:03,361 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9806b0fa8f18a18b9b918dba8c42bc67, REOPEN/MOVE because future has completed 2024-11-25T12:58:03,362 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-25T12:58:03,362 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246}] 2024-11-25T12:58:03,515 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(122): Close 9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:03,515 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-25T12:58:03,515 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1722): Closing 9806b0fa8f18a18b9b918dba8c42bc67, disabling compactions & flushes 2024-11-25T12:58:03,515 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:03,515 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:03,515 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. after waiting 0 ms 2024-11-25T12:58:03,515 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:03,519 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-25T12:58:03,519 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:03,520 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1676): Region close journal for 9806b0fa8f18a18b9b918dba8c42bc67: Waiting for close lock at 1732539483515Running coprocessor pre-close hooks at 1732539483515Disabling compacts and flushes for region at 1732539483515Disabling writes for close at 1732539483515Writing region close event to WAL at 1732539483515Running coprocessor post-close hooks at 1732539483519 (+4 ms)Closed at 1732539483519 2024-11-25T12:58:03,520 WARN [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegionServer(3299): Not adding moved region record: 9806b0fa8f18a18b9b918dba8c42bc67 to self. 2024-11-25T12:58:03,522 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(157): Closed 9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:03,522 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=9806b0fa8f18a18b9b918dba8c42bc67, regionState=CLOSED 2024-11-25T12:58:03,524 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 because future has completed 2024-11-25T12:58:03,526 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-25T12:58:03,526 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 in 163 msec 2024-11-25T12:58:03,526 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9806b0fa8f18a18b9b918dba8c42bc67, REOPEN/MOVE; state=CLOSED, location=1ff8009e463e,35905,1732539449246; forceNewPlan=false, retain=true 2024-11-25T12:58:03,677 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=9806b0fa8f18a18b9b918dba8c42bc67, regionState=OPENING, regionLocation=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:03,679 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9806b0fa8f18a18b9b918dba8c42bc67, REOPEN/MOVE because future has completed 2024-11-25T12:58:03,679 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246}] 2024-11-25T12:58:03,834 INFO [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:03,835 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} 2024-11-25T12:58:03,835 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:03,835 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:58:03,835 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for 9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:03,835 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for 9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:03,842 INFO [StoreOpener-9806b0fa8f18a18b9b918dba8c42bc67-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:03,848 INFO [StoreOpener-9806b0fa8f18a18b9b918dba8c42bc67-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9806b0fa8f18a18b9b918dba8c42bc67 columnFamilyName A 2024-11-25T12:58:03,854 DEBUG [StoreOpener-9806b0fa8f18a18b9b918dba8c42bc67-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:03,854 INFO [StoreOpener-9806b0fa8f18a18b9b918dba8c42bc67-1 {}] regionserver.HStore(327): Store=9806b0fa8f18a18b9b918dba8c42bc67/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:58:03,855 INFO [StoreOpener-9806b0fa8f18a18b9b918dba8c42bc67-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:03,856 INFO [StoreOpener-9806b0fa8f18a18b9b918dba8c42bc67-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9806b0fa8f18a18b9b918dba8c42bc67 columnFamilyName B 2024-11-25T12:58:03,856 DEBUG [StoreOpener-9806b0fa8f18a18b9b918dba8c42bc67-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:03,857 INFO [StoreOpener-9806b0fa8f18a18b9b918dba8c42bc67-1 {}] regionserver.HStore(327): Store=9806b0fa8f18a18b9b918dba8c42bc67/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:58:03,857 INFO [StoreOpener-9806b0fa8f18a18b9b918dba8c42bc67-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:03,858 INFO [StoreOpener-9806b0fa8f18a18b9b918dba8c42bc67-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9806b0fa8f18a18b9b918dba8c42bc67 columnFamilyName C 2024-11-25T12:58:03,858 DEBUG [StoreOpener-9806b0fa8f18a18b9b918dba8c42bc67-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:03,858 INFO [StoreOpener-9806b0fa8f18a18b9b918dba8c42bc67-1 {}] regionserver.HStore(327): Store=9806b0fa8f18a18b9b918dba8c42bc67/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:58:03,859 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for 9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:03,863 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:03,865 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:03,866 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for 9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:03,866 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for 9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:03,867 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-25T12:58:03,869 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for 9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:03,870 INFO [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened 9806b0fa8f18a18b9b918dba8c42bc67; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63254762, jitterRate=-0.05743059515953064}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-25T12:58:03,870 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:03,871 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for 9806b0fa8f18a18b9b918dba8c42bc67: Running coprocessor pre-open hook at 1732539483835Writing region info on filesystem at 1732539483835Initializing all the Stores at 1732539483839 (+4 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732539483839Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732539483839Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732539483839Cleaning up temporary data from old regions at 1732539483866 (+27 ms)Running coprocessor post-open hooks at 1732539483870 (+4 ms)Region opened successfully at 1732539483871 (+1 ms) 2024-11-25T12:58:03,879 INFO [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., pid=35, masterSystemTime=1732539483831 2024-11-25T12:58:03,892 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:03,892 INFO [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:03,892 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=9806b0fa8f18a18b9b918dba8c42bc67, regionState=OPEN, openSeqNum=5, regionLocation=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:03,894 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 because future has completed 2024-11-25T12:58:03,897 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=33 2024-11-25T12:58:03,897 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure 9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 in 216 msec 2024-11-25T12:58:03,898 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-25T12:58:03,899 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9806b0fa8f18a18b9b918dba8c42bc67, REOPEN/MOVE in 539 msec 2024-11-25T12:58:03,902 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-25T12:58:03,902 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 550 msec 2024-11-25T12:58:03,907 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 613 msec 2024-11-25T12:58:03,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-25T12:58:03,910 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:58:03,910 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@681c9b14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:58:03,910 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ff8009e463e,46289,-1 for getting cluster id 2024-11-25T12:58:03,911 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:58:03,912 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb972fd5-cdcb-422b-acfe-41fc80d301d9' 2024-11-25T12:58:03,912 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:58:03,912 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb972fd5-cdcb-422b-acfe-41fc80d301d9" 2024-11-25T12:58:03,913 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3efa15e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:58:03,913 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ff8009e463e,46289,-1] 2024-11-25T12:58:03,913 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:58:03,913 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:58:03,914 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42162, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:58:03,915 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e4f64ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:58:03,916 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51d91e8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:58:03,916 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ff8009e463e,46289,-1 for getting cluster id 2024-11-25T12:58:03,916 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:58:03,917 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb972fd5-cdcb-422b-acfe-41fc80d301d9' 2024-11-25T12:58:03,917 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:58:03,917 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb972fd5-cdcb-422b-acfe-41fc80d301d9" 2024-11-25T12:58:03,918 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c07527f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:58:03,918 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ff8009e463e,46289,-1] 2024-11-25T12:58:03,918 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:58:03,918 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:58:03,919 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42178, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:58:03,919 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67d312dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:58:03,920 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@85a5f7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:58:03,920 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ff8009e463e,46289,-1 for getting cluster id 2024-11-25T12:58:03,920 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:58:03,921 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb972fd5-cdcb-422b-acfe-41fc80d301d9' 2024-11-25T12:58:03,921 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:58:03,921 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb972fd5-cdcb-422b-acfe-41fc80d301d9" 2024-11-25T12:58:03,922 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@355f05d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:58:03,922 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ff8009e463e,46289,-1] 2024-11-25T12:58:03,922 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:58:03,922 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:58:03,923 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42186, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:58:03,924 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50dc9315, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:58:03,924 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37b75e2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:58:03,924 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ff8009e463e,46289,-1 for getting cluster id 2024-11-25T12:58:03,925 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:58:03,925 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb972fd5-cdcb-422b-acfe-41fc80d301d9' 2024-11-25T12:58:03,925 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:58:03,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb972fd5-cdcb-422b-acfe-41fc80d301d9" 2024-11-25T12:58:03,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f1a5352, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:58:03,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ff8009e463e,46289,-1] 2024-11-25T12:58:03,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:58:03,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:58:03,927 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42212, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:58:03,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@208d33f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:58:03,928 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e3f6eb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:58:03,928 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ff8009e463e,46289,-1 for getting cluster id 2024-11-25T12:58:03,928 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:58:03,929 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb972fd5-cdcb-422b-acfe-41fc80d301d9' 2024-11-25T12:58:03,929 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:58:03,929 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb972fd5-cdcb-422b-acfe-41fc80d301d9" 2024-11-25T12:58:03,929 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22d9f79, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:58:03,929 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ff8009e463e,46289,-1] 2024-11-25T12:58:03,929 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:58:03,929 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:58:03,930 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42236, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:58:03,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a9f3fd2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:58:03,932 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73b6e690, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:58:03,932 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ff8009e463e,46289,-1 for getting cluster id 2024-11-25T12:58:03,932 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:58:03,933 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb972fd5-cdcb-422b-acfe-41fc80d301d9' 2024-11-25T12:58:03,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:58:03,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb972fd5-cdcb-422b-acfe-41fc80d301d9" 2024-11-25T12:58:03,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@508b7d8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:58:03,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ff8009e463e,46289,-1] 2024-11-25T12:58:03,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:58:03,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:58:03,934 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42256, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:58:03,935 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27b4273e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:58:03,939 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bd05ea8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:58:03,940 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ff8009e463e,46289,-1 for getting cluster id 2024-11-25T12:58:03,940 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:58:03,941 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb972fd5-cdcb-422b-acfe-41fc80d301d9' 2024-11-25T12:58:03,941 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:58:03,941 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb972fd5-cdcb-422b-acfe-41fc80d301d9" 2024-11-25T12:58:03,941 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74ebe663, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:58:03,941 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ff8009e463e,46289,-1] 2024-11-25T12:58:03,942 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:58:03,942 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:58:03,943 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42270, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:58:03,943 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@604865ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:58:03,947 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dff183a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:58:03,947 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ff8009e463e,46289,-1 for getting cluster id 2024-11-25T12:58:03,947 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:58:03,948 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb972fd5-cdcb-422b-acfe-41fc80d301d9' 2024-11-25T12:58:03,948 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:58:03,949 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb972fd5-cdcb-422b-acfe-41fc80d301d9" 2024-11-25T12:58:03,949 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c438738, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:58:03,949 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ff8009e463e,46289,-1] 2024-11-25T12:58:03,949 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:58:03,949 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:58:03,950 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42292, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:58:03,950 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@630d23ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:58:03,951 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2874ba29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:58:03,951 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ff8009e463e,46289,-1 for getting cluster id 2024-11-25T12:58:03,951 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:58:03,953 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb972fd5-cdcb-422b-acfe-41fc80d301d9' 2024-11-25T12:58:03,953 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:58:03,953 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb972fd5-cdcb-422b-acfe-41fc80d301d9" 2024-11-25T12:58:03,953 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29232e75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:58:03,953 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ff8009e463e,46289,-1] 2024-11-25T12:58:03,953 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:58:03,954 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:58:03,955 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42308, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:58:03,956 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f1a2f4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:58:03,960 DEBUG [Thread-569 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:58:03,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,35905,1732539449246, seqNum=-1] 2024-11-25T12:58:03,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:58:03,966 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59944, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:58:03,970 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5] 2024-11-25T12:58:03,971 DEBUG [Thread-578 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:58:03,972 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,35905,1732539449246, seqNum=-1] 2024-11-25T12:58:03,973 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:58:03,975 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59948, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:58:03,978 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5] 2024-11-25T12:58:03,988 DEBUG [Thread-571 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:58:03,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,35905,1732539449246, seqNum=-1] 2024-11-25T12:58:03,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:58:03,991 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59962, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:58:03,994 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5] 2024-11-25T12:58:04,003 DEBUG [Thread-574 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:58:04,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,35905,1732539449246, seqNum=-1] 2024-11-25T12:58:04,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:58:04,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:04,007 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:58:04,007 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9806b0fa8f18a18b9b918dba8c42bc67 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-25T12:58:04,012 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:58:04,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees 2024-11-25T12:58:04,015 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:58:04,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-25T12:58:04,019 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:58:04,019 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:58:04,019 DEBUG [Thread-567 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:58:04,026 DEBUG [Thread-580 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:58:04,028 DEBUG [Thread-563 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:58:04,028 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,35905,1732539449246, seqNum=-1] 2024-11-25T12:58:04,029 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,35905,1732539449246, seqNum=-1] 2024-11-25T12:58:04,030 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:58:04,030 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,35905,1732539449246, seqNum=-1] 2024-11-25T12:58:04,030 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:58:04,032 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59966, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:58:04,032 DEBUG [Thread-580 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:58:04,034 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59968, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:58:04,038 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5] 2024-11-25T12:58:04,040 DEBUG [Thread-565 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:58:04,040 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59978, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:58:04,040 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5] 2024-11-25T12:58:04,041 DEBUG [Thread-576 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:58:04,042 DEBUG [Thread-576 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,35905,1732539449246, seqNum=-1] 2024-11-25T12:58:04,042 DEBUG [Thread-576 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:58:04,047 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59992, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:58:04,049 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60006, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:58:04,049 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5] 2024-11-25T12:58:04,051 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,35905,1732539449246, seqNum=-1] 2024-11-25T12:58:04,051 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:58:04,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5] 2024-11-25T12:58:04,052 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60008, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:58:04,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5] 2024-11-25T12:58:04,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5] 2024-11-25T12:58:04,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-25T12:58:04,131 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125079e2a32619647739a5cb9c6a8b83e83_9806b0fa8f18a18b9b918dba8c42bc67 is 50, key is test_row_0/A:col10/1732539484004/Put/seqid=0 2024-11-25T12:58:04,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:04,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59962 deadline: 1732539544125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:04,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:04,139 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:04,139 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:04,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60008 deadline: 1732539544127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:04,139 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:04,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:04,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:04,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:04,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:04,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59944 deadline: 1732539544131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:04,144 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:04,144 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:04,144 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:04,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:04,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59966 deadline: 1732539544135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:04,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:04,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:04,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:04,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:04,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59968 deadline: 1732539544137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:04,156 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:04,156 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:04,156 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:04,176 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-25T12:58:04,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:04,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. as already flushing 2024-11-25T12:58:04,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:04,177 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:04,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:04,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:04,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741933_1109 (size=12154) 2024-11-25T12:58:04,222 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:04,228 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125079e2a32619647739a5cb9c6a8b83e83_9806b0fa8f18a18b9b918dba8c42bc67 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125079e2a32619647739a5cb9c6a8b83e83_9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:04,239 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/9bf62f7391f440abb5089020f45c6f80, store: [table=TestAcidGuarantees family=A region=9806b0fa8f18a18b9b918dba8c42bc67] 2024-11-25T12:58:04,255 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/9bf62f7391f440abb5089020f45c6f80 is 175, key is test_row_0/A:col10/1732539484004/Put/seqid=0 2024-11-25T12:58:04,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:04,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59962 deadline: 1732539544250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:04,264 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:04,264 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:04,264 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:04,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:04,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59944 deadline: 1732539544251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:04,267 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:04,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:04,267 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:04,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59966 deadline: 1732539544251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:04,267 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:04,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:04,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60008 deadline: 1732539544252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:04,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:04,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:04,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:04,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:04,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:04,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:04,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:04,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59968 deadline: 1732539544264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:04,273 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:04,273 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:04,273 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:04,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741934_1110 (size=30955) 2024-11-25T12:58:04,296 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/9bf62f7391f440abb5089020f45c6f80 2024-11-25T12:58:04,332 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-25T12:58:04,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:04,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. as already flushing 2024-11-25T12:58:04,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:04,333 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:04,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:04,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-25T12:58:04,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:04,350 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/B/5f4a1ef9f4eb42c08ad186f6bcac23c7 is 50, key is test_row_0/B:col10/1732539484004/Put/seqid=0 2024-11-25T12:58:04,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741935_1111 (size=12001) 2024-11-25T12:58:04,387 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/B/5f4a1ef9f4eb42c08ad186f6bcac23c7 2024-11-25T12:58:04,446 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/C/1545a41de1ce4180a663b971f0436b02 is 50, key is test_row_0/C:col10/1732539484004/Put/seqid=0 2024-11-25T12:58:04,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:04,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59944 deadline: 1732539544471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:04,483 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:04,483 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:04,483 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:04,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:04,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59962 deadline: 1732539544471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:04,485 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:04,485 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:04,485 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:04,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741936_1112 (size=12001) 2024-11-25T12:58:04,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:04,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60008 deadline: 1732539544480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:04,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:04,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:04,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:04,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:04,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59966 deadline: 1732539544480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:04,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:04,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59968 deadline: 1732539544482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:04,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:04,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:04,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:04,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:04,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:04,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:04,500 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-25T12:58:04,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:04,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. as already flushing 2024-11-25T12:58:04,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:04,500 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:04,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:04,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:04,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-25T12:58:04,656 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-25T12:58:04,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:04,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. as already flushing 2024-11-25T12:58:04,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:04,657 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:04,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:04,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:04,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:04,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59944 deadline: 1732539544790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:04,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:04,795 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:04,795 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:04,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59962 deadline: 1732539544791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:04,795 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:04,796 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:04,796 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:04,796 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:04,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:04,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59968 deadline: 1732539544800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:04,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:04,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:04,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:04,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:04,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60008 deadline: 1732539544800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:04,811 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-25T12:58:04,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:04,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:04,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:04,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:04,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. as already flushing 2024-11-25T12:58:04,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:04,812 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:04,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:04,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:04,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59966 deadline: 1732539544810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:04,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:04,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:04,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:04,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:04,900 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/C/1545a41de1ce4180a663b971f0436b02 2024-11-25T12:58:04,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/9bf62f7391f440abb5089020f45c6f80 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/9bf62f7391f440abb5089020f45c6f80 2024-11-25T12:58:04,930 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/9bf62f7391f440abb5089020f45c6f80, entries=150, sequenceid=15, filesize=30.2 K 2024-11-25T12:58:04,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/B/5f4a1ef9f4eb42c08ad186f6bcac23c7 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/5f4a1ef9f4eb42c08ad186f6bcac23c7 2024-11-25T12:58:04,941 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/5f4a1ef9f4eb42c08ad186f6bcac23c7, entries=150, sequenceid=15, filesize=11.7 K 2024-11-25T12:58:04,944 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/C/1545a41de1ce4180a663b971f0436b02 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/1545a41de1ce4180a663b971f0436b02 2024-11-25T12:58:04,954 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/1545a41de1ce4180a663b971f0436b02, entries=150, sequenceid=15, filesize=11.7 K 2024-11-25T12:58:04,956 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 9806b0fa8f18a18b9b918dba8c42bc67 in 949ms, sequenceid=15, compaction requested=false 2024-11-25T12:58:04,956 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-25T12:58:04,958 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9806b0fa8f18a18b9b918dba8c42bc67: 2024-11-25T12:58:04,967 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-25T12:58:04,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:04,967 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2902): Flushing 9806b0fa8f18a18b9b918dba8c42bc67 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-25T12:58:04,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125e196fa7c773a4c16b513b855daf41b32_9806b0fa8f18a18b9b918dba8c42bc67 is 50, key is test_row_0/A:col10/1732539484103/Put/seqid=0 2024-11-25T12:58:05,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741937_1113 (size=12154) 2024-11-25T12:58:05,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:05,066 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125e196fa7c773a4c16b513b855daf41b32_9806b0fa8f18a18b9b918dba8c42bc67 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125e196fa7c773a4c16b513b855daf41b32_9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:05,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/42deca4a7a7a443b8807f66c48a02425, store: [table=TestAcidGuarantees family=A region=9806b0fa8f18a18b9b918dba8c42bc67] 2024-11-25T12:58:05,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/42deca4a7a7a443b8807f66c48a02425 is 175, key is test_row_0/A:col10/1732539484103/Put/seqid=0 2024-11-25T12:58:05,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741938_1114 (size=30955) 2024-11-25T12:58:05,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-25T12:58:05,310 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. as already flushing 2024-11-25T12:58:05,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:05,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:05,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59944 deadline: 1732539545342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:05,348 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:05,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:05,349 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:05,349 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:05,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59962 deadline: 1732539545343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:05,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:05,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60008 deadline: 1732539545343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:05,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:05,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:05,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:05,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:05,350 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:05,351 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:05,351 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:05,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59966 deadline: 1732539545345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:05,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:05,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:05,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59968 deadline: 1732539545345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:05,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:05,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:05,355 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:05,355 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:05,355 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:05,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:05,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59944 deadline: 1732539545460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:05,469 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:05,469 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:05,469 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:05,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:05,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:05,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59966 deadline: 1732539545460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:05,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60008 deadline: 1732539545460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:05,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:05,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:05,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:05,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:05,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59968 deadline: 1732539545461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:05,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:05,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:05,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:05,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:05,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:05,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:05,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:05,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59962 deadline: 1732539545461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:05,471 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:05,471 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:05,471 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:05,516 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/42deca4a7a7a443b8807f66c48a02425 2024-11-25T12:58:05,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/B/50b421bbb831456bafdf501a2a314b19 is 50, key is test_row_0/B:col10/1732539484103/Put/seqid=0 2024-11-25T12:58:05,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741939_1115 (size=12001) 2024-11-25T12:58:05,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:05,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59966 deadline: 1732539545681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:05,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:05,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60008 deadline: 1732539545681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:05,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:05,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:05,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:05,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:05,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59944 deadline: 1732539545682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:05,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:05,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59962 deadline: 1732539545682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:05,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:05,690 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:05,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59968 deadline: 1732539545682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:05,690 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:05,690 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:05,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:05,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:05,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:05,691 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:05,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:05,691 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:05,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:05,691 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:05,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:06,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:06,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60008 deadline: 1732539546000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:06,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:06,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:06,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:06,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59944 deadline: 1732539546000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:06,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:06,012 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:06,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:06,012 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:06,012 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:06,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59968 deadline: 1732539546001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:06,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:06,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59966 deadline: 1732539546001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:06,012 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:06,013 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:06,013 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:06,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:06,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59962 deadline: 1732539546003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:06,013 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:06,013 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:06,013 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:06,013 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:06,013 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:06,013 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:06,040 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/B/50b421bbb831456bafdf501a2a314b19 2024-11-25T12:58:06,081 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T12:58:06,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/C/782d48a2df4945caa79383ba393df85a is 50, key is test_row_0/C:col10/1732539484103/Put/seqid=0 2024-11-25T12:58:06,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741940_1116 (size=12001) 2024-11-25T12:58:06,156 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/C/782d48a2df4945caa79383ba393df85a 2024-11-25T12:58:06,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/42deca4a7a7a443b8807f66c48a02425 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/42deca4a7a7a443b8807f66c48a02425 2024-11-25T12:58:06,170 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/42deca4a7a7a443b8807f66c48a02425, entries=150, sequenceid=41, filesize=30.2 K 2024-11-25T12:58:06,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/B/50b421bbb831456bafdf501a2a314b19 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/50b421bbb831456bafdf501a2a314b19 2024-11-25T12:58:06,178 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/50b421bbb831456bafdf501a2a314b19, entries=150, sequenceid=41, filesize=11.7 K 2024-11-25T12:58:06,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/C/782d48a2df4945caa79383ba393df85a as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/782d48a2df4945caa79383ba393df85a 2024-11-25T12:58:06,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-25T12:58:06,186 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/782d48a2df4945caa79383ba393df85a, entries=150, sequenceid=41, filesize=11.7 K 2024-11-25T12:58:06,188 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for 9806b0fa8f18a18b9b918dba8c42bc67 in 1221ms, sequenceid=41, compaction requested=false 2024-11-25T12:58:06,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2603): Flush status journal for 9806b0fa8f18a18b9b918dba8c42bc67: 2024-11-25T12:58:06,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:06,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=37 2024-11-25T12:58:06,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4169): Remote procedure done, pid=37 2024-11-25T12:58:06,192 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-25T12:58:06,192 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1710 sec 2024-11-25T12:58:06,204 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees in 2.1810 sec 2024-11-25T12:58:06,525 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:58:06,525 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9806b0fa8f18a18b9b918dba8c42bc67 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-25T12:58:06,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:06,546 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411253ca5aaf3761f4bb4a946505051d92581_9806b0fa8f18a18b9b918dba8c42bc67 is 50, key is test_row_0/A:col10/1732539485331/Put/seqid=0 2024-11-25T12:58:06,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:06,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59944 deadline: 1732539546573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:06,582 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:06,583 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:06,583 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:06,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:06,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59962 deadline: 1732539546576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:06,584 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:06,584 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:06,584 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:06,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:06,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59968 deadline: 1732539546579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:06,585 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:06,585 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:06,585 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:06,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:06,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60008 deadline: 1732539546580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:06,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:06,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:06,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59966 deadline: 1732539546581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:06,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:06,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:06,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:06,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:06,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:06,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741941_1117 (size=14594) 2024-11-25T12:58:06,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:06,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60008 deadline: 1732539546690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:06,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:06,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:06,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:06,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:06,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59944 deadline: 1732539546691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:06,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:06,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:06,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59966 deadline: 1732539546691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:06,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59962 deadline: 1732539546690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:06,694 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:06,694 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:06,694 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:06,696 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:06,696 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:06,696 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:06,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:06,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:06,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:06,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:06,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59968 deadline: 1732539546695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:06,702 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:06,702 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:06,702 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:06,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:06,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60008 deadline: 1732539546900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:06,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:06,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59962 deadline: 1732539546900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:06,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:06,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:06,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:06,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:06,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59944 deadline: 1732539546901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:06,905 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:06,905 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:06,905 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:06,905 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:06,905 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:06,905 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:06,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:06,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59966 deadline: 1732539546901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:06,906 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:06,906 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:06,906 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:06,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:06,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59968 deadline: 1732539546911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:06,914 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:06,914 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:06,914 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:06,990 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:06,998 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411253ca5aaf3761f4bb4a946505051d92581_9806b0fa8f18a18b9b918dba8c42bc67 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411253ca5aaf3761f4bb4a946505051d92581_9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:07,000 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/76274d615dc649169b1980a5d941b129, store: [table=TestAcidGuarantees family=A region=9806b0fa8f18a18b9b918dba8c42bc67] 2024-11-25T12:58:07,001 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/76274d615dc649169b1980a5d941b129 is 175, key is test_row_0/A:col10/1732539485331/Put/seqid=0 2024-11-25T12:58:07,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741942_1118 (size=39549) 2024-11-25T12:58:07,020 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=53, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/76274d615dc649169b1980a5d941b129 2024-11-25T12:58:07,058 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/B/7f273fcdf56341b3af1ae9a8a1851e64 is 50, key is test_row_0/B:col10/1732539485331/Put/seqid=0 2024-11-25T12:58:07,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741943_1119 (size=12001) 2024-11-25T12:58:07,097 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/B/7f273fcdf56341b3af1ae9a8a1851e64 2024-11-25T12:58:07,122 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/C/8e9fc41ed9454d60ab1ed6690447bb77 is 50, key is test_row_0/C:col10/1732539485331/Put/seqid=0 2024-11-25T12:58:07,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741944_1120 (size=12001) 2024-11-25T12:58:07,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:07,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60008 deadline: 1732539547212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:07,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:07,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59962 deadline: 1732539547212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:07,217 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:07,217 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:07,217 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:07,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:07,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:07,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:07,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:07,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59966 deadline: 1732539547219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:07,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:07,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59944 deadline: 1732539547219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:07,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:07,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59968 deadline: 1732539547225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:07,227 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:07,227 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:07,227 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:07,229 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:07,229 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:07,229 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:07,230 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:07,230 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:07,230 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:07,555 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/C/8e9fc41ed9454d60ab1ed6690447bb77 2024-11-25T12:58:07,563 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/76274d615dc649169b1980a5d941b129 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/76274d615dc649169b1980a5d941b129 2024-11-25T12:58:07,569 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/76274d615dc649169b1980a5d941b129, entries=200, sequenceid=53, filesize=38.6 K 2024-11-25T12:58:07,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/B/7f273fcdf56341b3af1ae9a8a1851e64 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/7f273fcdf56341b3af1ae9a8a1851e64 2024-11-25T12:58:07,594 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/7f273fcdf56341b3af1ae9a8a1851e64, entries=150, sequenceid=53, filesize=11.7 K 2024-11-25T12:58:07,595 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/C/8e9fc41ed9454d60ab1ed6690447bb77 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/8e9fc41ed9454d60ab1ed6690447bb77 2024-11-25T12:58:07,605 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/8e9fc41ed9454d60ab1ed6690447bb77, entries=150, sequenceid=53, filesize=11.7 K 2024-11-25T12:58:07,616 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 9806b0fa8f18a18b9b918dba8c42bc67 in 1091ms, sequenceid=53, compaction requested=true 2024-11-25T12:58:07,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9806b0fa8f18a18b9b918dba8c42bc67: 2024-11-25T12:58:07,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9806b0fa8f18a18b9b918dba8c42bc67:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:58:07,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:58:07,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9806b0fa8f18a18b9b918dba8c42bc67:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:58:07,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-25T12:58:07,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9806b0fa8f18a18b9b918dba8c42bc67:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:58:07,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-25T12:58:07,617 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:58:07,619 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:58:07,629 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:58:07,630 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): 9806b0fa8f18a18b9b918dba8c42bc67/A is initiating minor compaction (all files) 2024-11-25T12:58:07,630 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9806b0fa8f18a18b9b918dba8c42bc67/A in TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:07,630 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/9bf62f7391f440abb5089020f45c6f80, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/42deca4a7a7a443b8807f66c48a02425, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/76274d615dc649169b1980a5d941b129] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp, totalSize=99.1 K 2024-11-25T12:58:07,630 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:07,630 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. files: [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/9bf62f7391f440abb5089020f45c6f80, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/42deca4a7a7a443b8807f66c48a02425, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/76274d615dc649169b1980a5d941b129] 2024-11-25T12:58:07,635 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 9bf62f7391f440abb5089020f45c6f80, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732539483996 2024-11-25T12:58:07,639 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:58:07,639 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 42deca4a7a7a443b8807f66c48a02425, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732539484103 2024-11-25T12:58:07,639 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1541): 9806b0fa8f18a18b9b918dba8c42bc67/C is initiating minor compaction (all files) 2024-11-25T12:58:07,640 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9806b0fa8f18a18b9b918dba8c42bc67/C in TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:07,640 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/1545a41de1ce4180a663b971f0436b02, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/782d48a2df4945caa79383ba393df85a, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/8e9fc41ed9454d60ab1ed6690447bb77] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp, totalSize=35.2 K 2024-11-25T12:58:07,643 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 76274d615dc649169b1980a5d941b129, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732539485331 2024-11-25T12:58:07,647 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1545a41de1ce4180a663b971f0436b02, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732539483996 2024-11-25T12:58:07,651 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 782d48a2df4945caa79383ba393df85a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732539484103 2024-11-25T12:58:07,659 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e9fc41ed9454d60ab1ed6690447bb77, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732539485331 2024-11-25T12:58:07,682 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9806b0fa8f18a18b9b918dba8c42bc67] 2024-11-25T12:58:07,708 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241125bf79eb83313342879ecb2a825f8529a0_9806b0fa8f18a18b9b918dba8c42bc67 store=[table=TestAcidGuarantees family=A region=9806b0fa8f18a18b9b918dba8c42bc67] 2024-11-25T12:58:07,712 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241125bf79eb83313342879ecb2a825f8529a0_9806b0fa8f18a18b9b918dba8c42bc67, store=[table=TestAcidGuarantees family=A region=9806b0fa8f18a18b9b918dba8c42bc67] 2024-11-25T12:58:07,713 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125bf79eb83313342879ecb2a825f8529a0_9806b0fa8f18a18b9b918dba8c42bc67 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9806b0fa8f18a18b9b918dba8c42bc67] 2024-11-25T12:58:07,722 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9806b0fa8f18a18b9b918dba8c42bc67#C#compaction#103 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:58:07,722 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/C/d240fdce83f94459be2a8f384cf3dc9d is 50, key is test_row_0/C:col10/1732539485331/Put/seqid=0 2024-11-25T12:58:07,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741945_1121 (size=4469) 2024-11-25T12:58:07,741 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9806b0fa8f18a18b9b918dba8c42bc67#A#compaction#102 average throughput is 0.42 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:58:07,741 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/45b4db3b2f954395ac6a5ad67fcbed6e is 175, key is test_row_0/A:col10/1732539485331/Put/seqid=0 2024-11-25T12:58:07,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:07,743 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:58:07,744 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9806b0fa8f18a18b9b918dba8c42bc67 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-25T12:58:07,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741946_1122 (size=12104) 2024-11-25T12:58:07,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741947_1123 (size=31058) 2024-11-25T12:58:07,817 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411256849d301e9fa48dea0b414779257cc40_9806b0fa8f18a18b9b918dba8c42bc67 is 50, key is test_row_0/A:col10/1732539486577/Put/seqid=0 2024-11-25T12:58:07,837 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/45b4db3b2f954395ac6a5ad67fcbed6e as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/45b4db3b2f954395ac6a5ad67fcbed6e 2024-11-25T12:58:07,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:07,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60008 deadline: 1732539547816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:07,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:07,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:07,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59968 deadline: 1732539547818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:07,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:07,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:07,842 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:07,842 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:07,842 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:07,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:07,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59944 deadline: 1732539547822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:07,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:07,843 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:07,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59962 deadline: 1732539547824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:07,843 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:07,843 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:07,843 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:07,843 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:07,843 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:07,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:07,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59966 deadline: 1732539547836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:07,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:07,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:07,845 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:07,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741948_1124 (size=12154) 2024-11-25T12:58:07,885 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:07,894 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9806b0fa8f18a18b9b918dba8c42bc67/A of 9806b0fa8f18a18b9b918dba8c42bc67 into 45b4db3b2f954395ac6a5ad67fcbed6e(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:58:07,894 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9806b0fa8f18a18b9b918dba8c42bc67: 2024-11-25T12:58:07,894 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., storeName=9806b0fa8f18a18b9b918dba8c42bc67/A, priority=13, startTime=1732539487616; duration=0sec 2024-11-25T12:58:07,894 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:58:07,894 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9806b0fa8f18a18b9b918dba8c42bc67:A 2024-11-25T12:58:07,896 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:58:07,901 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:58:07,901 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): 9806b0fa8f18a18b9b918dba8c42bc67/B is initiating minor compaction (all files) 2024-11-25T12:58:07,901 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9806b0fa8f18a18b9b918dba8c42bc67/B in TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:07,901 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/5f4a1ef9f4eb42c08ad186f6bcac23c7, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/50b421bbb831456bafdf501a2a314b19, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/7f273fcdf56341b3af1ae9a8a1851e64] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp, totalSize=35.2 K 2024-11-25T12:58:07,904 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f4a1ef9f4eb42c08ad186f6bcac23c7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732539483996 2024-11-25T12:58:07,906 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411256849d301e9fa48dea0b414779257cc40_9806b0fa8f18a18b9b918dba8c42bc67 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411256849d301e9fa48dea0b414779257cc40_9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:07,907 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 50b421bbb831456bafdf501a2a314b19, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732539484103 2024-11-25T12:58:07,907 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f273fcdf56341b3af1ae9a8a1851e64, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732539485331 2024-11-25T12:58:07,907 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/a4e1a73720dd40e8a3fe8d17add0b9ee, store: [table=TestAcidGuarantees family=A region=9806b0fa8f18a18b9b918dba8c42bc67] 2024-11-25T12:58:07,908 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/a4e1a73720dd40e8a3fe8d17add0b9ee is 175, key is test_row_0/A:col10/1732539486577/Put/seqid=0 2024-11-25T12:58:07,937 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9806b0fa8f18a18b9b918dba8c42bc67#B#compaction#105 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:58:07,938 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/B/abb9530cb5324d9fb6e927527e32e0e1 is 50, key is test_row_0/B:col10/1732539485331/Put/seqid=0 2024-11-25T12:58:07,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:07,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59944 deadline: 1732539547951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:07,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:07,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59968 deadline: 1732539547951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:07,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:07,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60008 deadline: 1732539547951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:07,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:07,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59966 deadline: 1732539547951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:07,959 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:07,959 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:07,959 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:07,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:07,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:07,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:07,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:07,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:07,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:07,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:07,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59962 deadline: 1732539547951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:07,960 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:07,960 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:07,960 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:07,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741949_1125 (size=30955) 2024-11-25T12:58:07,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:07,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:07,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:07,968 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=79, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/a4e1a73720dd40e8a3fe8d17add0b9ee 2024-11-25T12:58:08,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741950_1126 (size=12104) 2024-11-25T12:58:08,014 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/B/021b38eb770f4bffb029885ed7a91a82 is 50, key is test_row_0/B:col10/1732539486577/Put/seqid=0 2024-11-25T12:58:08,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741951_1127 (size=12001) 2024-11-25T12:58:08,095 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/B/021b38eb770f4bffb029885ed7a91a82 2024-11-25T12:58:08,137 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/C/19973b88a14e4e17ab77d2f88fd60538 is 50, key is test_row_0/C:col10/1732539486577/Put/seqid=0 2024-11-25T12:58:08,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:08,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59962 deadline: 1732539548170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:08,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:08,178 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:08,178 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:08,179 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:08,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:08,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60008 deadline: 1732539548171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:08,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59968 deadline: 1732539548171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:08,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:08,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:08,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:08,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:08,180 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:08,180 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:08,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59966 deadline: 1732539548176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:08,180 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:08,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:08,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:08,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:08,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59944 deadline: 1732539548176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:08,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:08,181 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:08,181 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:08,181 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:08,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741952_1128 (size=12001) 2024-11-25T12:58:08,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-25T12:58:08,193 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:58:08,194 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:58:08,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-25T12:58:08,196 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:58:08,200 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:58:08,200 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:58:08,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-25T12:58:08,218 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/C/d240fdce83f94459be2a8f384cf3dc9d as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/d240fdce83f94459be2a8f384cf3dc9d 2024-11-25T12:58:08,242 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9806b0fa8f18a18b9b918dba8c42bc67/C of 9806b0fa8f18a18b9b918dba8c42bc67 into d240fdce83f94459be2a8f384cf3dc9d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:58:08,242 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9806b0fa8f18a18b9b918dba8c42bc67: 2024-11-25T12:58:08,242 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., storeName=9806b0fa8f18a18b9b918dba8c42bc67/C, priority=13, startTime=1732539487617; duration=0sec 2024-11-25T12:58:08,242 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:58:08,242 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9806b0fa8f18a18b9b918dba8c42bc67:C 2024-11-25T12:58:08,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-25T12:58:08,356 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-25T12:58:08,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:08,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. as already flushing 2024-11-25T12:58:08,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:08,358 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:08,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:08,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:08,415 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/B/abb9530cb5324d9fb6e927527e32e0e1 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/abb9530cb5324d9fb6e927527e32e0e1 2024-11-25T12:58:08,428 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9806b0fa8f18a18b9b918dba8c42bc67/B of 9806b0fa8f18a18b9b918dba8c42bc67 into abb9530cb5324d9fb6e927527e32e0e1(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:58:08,428 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9806b0fa8f18a18b9b918dba8c42bc67: 2024-11-25T12:58:08,428 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., storeName=9806b0fa8f18a18b9b918dba8c42bc67/B, priority=13, startTime=1732539487616; duration=0sec 2024-11-25T12:58:08,428 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:58:08,428 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9806b0fa8f18a18b9b918dba8c42bc67:B 2024-11-25T12:58:08,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:08,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60008 deadline: 1732539548491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:08,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:08,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:08,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59962 deadline: 1732539548491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:08,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:08,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:08,494 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:08,494 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:08,494 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:08,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:08,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59966 deadline: 1732539548491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:08,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:08,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:08,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:08,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:08,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59944 deadline: 1732539548491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:08,495 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:08,495 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:08,496 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:08,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:08,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59968 deadline: 1732539548491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:08,498 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:08,498 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:08,498 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:08,513 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-25T12:58:08,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:08,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. as already flushing 2024-11-25T12:58:08,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:08,513 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:08,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:08,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:08,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-25T12:58:08,593 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/C/19973b88a14e4e17ab77d2f88fd60538 2024-11-25T12:58:08,600 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/a4e1a73720dd40e8a3fe8d17add0b9ee as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/a4e1a73720dd40e8a3fe8d17add0b9ee 2024-11-25T12:58:08,607 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/a4e1a73720dd40e8a3fe8d17add0b9ee, entries=150, sequenceid=79, filesize=30.2 K 2024-11-25T12:58:08,609 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/B/021b38eb770f4bffb029885ed7a91a82 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/021b38eb770f4bffb029885ed7a91a82 2024-11-25T12:58:08,633 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/021b38eb770f4bffb029885ed7a91a82, entries=150, sequenceid=79, filesize=11.7 K 2024-11-25T12:58:08,643 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/C/19973b88a14e4e17ab77d2f88fd60538 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/19973b88a14e4e17ab77d2f88fd60538 2024-11-25T12:58:08,666 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-25T12:58:08,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:08,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. as already flushing 2024-11-25T12:58:08,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:08,666 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:08,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:08,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:08,680 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/19973b88a14e4e17ab77d2f88fd60538, entries=150, sequenceid=79, filesize=11.7 K 2024-11-25T12:58:08,684 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=60.38 KB/61830 for 9806b0fa8f18a18b9b918dba8c42bc67 in 941ms, sequenceid=79, compaction requested=false 2024-11-25T12:58:08,684 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9806b0fa8f18a18b9b918dba8c42bc67: 2024-11-25T12:58:08,828 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-25T12:58:08,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:08,828 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing 9806b0fa8f18a18b9b918dba8c42bc67 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-25T12:58:08,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-25T12:58:08,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112582f6b93d490048bcaad54b78ba8095c0_9806b0fa8f18a18b9b918dba8c42bc67 is 50, key is test_row_0/A:col10/1732539487791/Put/seqid=0 2024-11-25T12:58:08,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741953_1129 (size=12154) 2024-11-25T12:58:08,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:08,933 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112582f6b93d490048bcaad54b78ba8095c0_9806b0fa8f18a18b9b918dba8c42bc67 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112582f6b93d490048bcaad54b78ba8095c0_9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:08,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/48f0f15698d041a0b35a39bcdfae5ae0, store: [table=TestAcidGuarantees family=A region=9806b0fa8f18a18b9b918dba8c42bc67] 2024-11-25T12:58:08,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/48f0f15698d041a0b35a39bcdfae5ae0 is 175, key is test_row_0/A:col10/1732539487791/Put/seqid=0 2024-11-25T12:58:08,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741954_1130 (size=30955) 2024-11-25T12:58:08,983 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=94, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/48f0f15698d041a0b35a39bcdfae5ae0 2024-11-25T12:58:09,007 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. as already flushing 2024-11-25T12:58:09,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:09,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/B/40a49f11b38d4dcf87eb05631338c71a is 50, key is test_row_0/B:col10/1732539487791/Put/seqid=0 2024-11-25T12:58:09,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741955_1131 (size=12001) 2024-11-25T12:58:09,085 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/B/40a49f11b38d4dcf87eb05631338c71a 2024-11-25T12:58:09,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/C/35804b79e49146b58ebfcd1146434bb5 is 50, key is test_row_0/C:col10/1732539487791/Put/seqid=0 2024-11-25T12:58:09,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:09,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59968 deadline: 1732539549091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:09,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:09,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59944 deadline: 1732539549102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:09,125 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:09,125 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:09,125 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:09,127 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:09,127 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:09,128 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:09,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:09,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59962 deadline: 1732539549105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:09,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:09,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59966 deadline: 1732539549116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:09,132 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:09,132 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:09,132 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:09,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:09,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:09,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:09,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741956_1132 (size=12001) 2024-11-25T12:58:09,146 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/C/35804b79e49146b58ebfcd1146434bb5 2024-11-25T12:58:09,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:09,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60008 deadline: 1732539549123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:09,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:09,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:09,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:09,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/48f0f15698d041a0b35a39bcdfae5ae0 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/48f0f15698d041a0b35a39bcdfae5ae0 2024-11-25T12:58:09,166 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/48f0f15698d041a0b35a39bcdfae5ae0, entries=150, sequenceid=94, filesize=30.2 K 2024-11-25T12:58:09,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/B/40a49f11b38d4dcf87eb05631338c71a as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/40a49f11b38d4dcf87eb05631338c71a 2024-11-25T12:58:09,176 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/40a49f11b38d4dcf87eb05631338c71a, entries=150, sequenceid=94, filesize=11.7 K 2024-11-25T12:58:09,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/C/35804b79e49146b58ebfcd1146434bb5 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/35804b79e49146b58ebfcd1146434bb5 2024-11-25T12:58:09,191 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/35804b79e49146b58ebfcd1146434bb5, entries=150, sequenceid=94, filesize=11.7 K 2024-11-25T12:58:09,197 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 9806b0fa8f18a18b9b918dba8c42bc67 in 368ms, sequenceid=94, compaction requested=true 2024-11-25T12:58:09,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for 9806b0fa8f18a18b9b918dba8c42bc67: 2024-11-25T12:58:09,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:09,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-25T12:58:09,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-25T12:58:09,202 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-25T12:58:09,202 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 999 msec 2024-11-25T12:58:09,204 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 1.0080 sec 2024-11-25T12:58:09,242 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:58:09,243 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9806b0fa8f18a18b9b918dba8c42bc67 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-25T12:58:09,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:09,264 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411253641b6e4a394435689092a05f886538e_9806b0fa8f18a18b9b918dba8c42bc67 is 50, key is test_row_0/A:col10/1732539489240/Put/seqid=0 2024-11-25T12:58:09,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:09,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59966 deadline: 1732539549279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:09,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:09,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:09,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:09,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:09,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59968 deadline: 1732539549280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:09,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:09,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60008 deadline: 1732539549284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:09,311 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:09,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:09,311 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:09,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:09,311 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:09,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:09,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741957_1133 (size=14594) 2024-11-25T12:58:09,316 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:09,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:09,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59944 deadline: 1732539549305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:09,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:09,318 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:09,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59962 deadline: 1732539549308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:09,318 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:09,318 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:09,319 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:09,319 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:09,319 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:09,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-25T12:58:09,343 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:58:09,343 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:58:09,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-25T12:58:09,346 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411253641b6e4a394435689092a05f886538e_9806b0fa8f18a18b9b918dba8c42bc67 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411253641b6e4a394435689092a05f886538e_9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:09,347 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:58:09,348 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/3ba02a1801164515b68ae50df492f529, store: [table=TestAcidGuarantees family=A region=9806b0fa8f18a18b9b918dba8c42bc67] 2024-11-25T12:58:09,348 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/3ba02a1801164515b68ae50df492f529 is 175, key is test_row_0/A:col10/1732539489240/Put/seqid=0 2024-11-25T12:58:09,348 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:58:09,349 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:58:09,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-25T12:58:09,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741958_1134 (size=39549) 2024-11-25T12:58:09,414 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=119, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/3ba02a1801164515b68ae50df492f529 2024-11-25T12:58:09,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:09,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59966 deadline: 1732539549415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:09,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:09,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:09,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:09,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:09,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60008 deadline: 1732539549421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:09,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:09,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:09,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:09,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:09,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59944 deadline: 1732539549421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:09,433 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:09,433 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:09,433 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:09,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:09,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59968 deadline: 1732539549423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:09,435 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:09,435 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:09,435 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:09,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:09,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59962 deadline: 1732539549430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:09,442 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:09,442 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:09,442 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:09,450 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/B/b73d79d4de1c4c7f9057aaf4adf92424 is 50, key is test_row_0/B:col10/1732539489240/Put/seqid=0 2024-11-25T12:58:09,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-25T12:58:09,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741959_1135 (size=12001) 2024-11-25T12:58:09,479 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/B/b73d79d4de1c4c7f9057aaf4adf92424 2024-11-25T12:58:09,509 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/C/046ce0ad39ae48eaa1206d3508685b7e is 50, key is test_row_0/C:col10/1732539489240/Put/seqid=0 2024-11-25T12:58:09,519 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-25T12:58:09,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:09,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. as already flushing 2024-11-25T12:58:09,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:09,519 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:09,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:09,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:09,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741960_1136 (size=12001) 2024-11-25T12:58:09,552 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/C/046ce0ad39ae48eaa1206d3508685b7e 2024-11-25T12:58:09,563 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/3ba02a1801164515b68ae50df492f529 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/3ba02a1801164515b68ae50df492f529 2024-11-25T12:58:09,583 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/3ba02a1801164515b68ae50df492f529, entries=200, sequenceid=119, filesize=38.6 K 2024-11-25T12:58:09,587 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/B/b73d79d4de1c4c7f9057aaf4adf92424 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/b73d79d4de1c4c7f9057aaf4adf92424 2024-11-25T12:58:09,594 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/b73d79d4de1c4c7f9057aaf4adf92424, entries=150, sequenceid=119, filesize=11.7 K 2024-11-25T12:58:09,601 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/C/046ce0ad39ae48eaa1206d3508685b7e as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/046ce0ad39ae48eaa1206d3508685b7e 2024-11-25T12:58:09,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:09,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59966 deadline: 1732539549635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:09,639 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/046ce0ad39ae48eaa1206d3508685b7e, entries=150, sequenceid=119, filesize=11.7 K 2024-11-25T12:58:09,645 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 9806b0fa8f18a18b9b918dba8c42bc67 in 401ms, sequenceid=119, compaction requested=true 2024-11-25T12:58:09,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9806b0fa8f18a18b9b918dba8c42bc67: 2024-11-25T12:58:09,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9806b0fa8f18a18b9b918dba8c42bc67:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:58:09,645 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:58:09,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:58:09,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9806b0fa8f18a18b9b918dba8c42bc67:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:58:09,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:58:09,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9806b0fa8f18a18b9b918dba8c42bc67:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:58:09,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-25T12:58:09,646 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:58:09,647 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132517 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:58:09,649 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:58:09,649 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): 9806b0fa8f18a18b9b918dba8c42bc67/B is initiating minor compaction (all files) 2024-11-25T12:58:09,650 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9806b0fa8f18a18b9b918dba8c42bc67/B in TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:09,650 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/abb9530cb5324d9fb6e927527e32e0e1, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/021b38eb770f4bffb029885ed7a91a82, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/40a49f11b38d4dcf87eb05631338c71a, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/b73d79d4de1c4c7f9057aaf4adf92424] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp, totalSize=47.0 K 2024-11-25T12:58:09,650 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting abb9530cb5324d9fb6e927527e32e0e1, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732539485331 2024-11-25T12:58:09,651 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1541): 9806b0fa8f18a18b9b918dba8c42bc67/A is initiating minor compaction (all files) 2024-11-25T12:58:09,651 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 021b38eb770f4bffb029885ed7a91a82, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732539486575 2024-11-25T12:58:09,651 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9806b0fa8f18a18b9b918dba8c42bc67/A in TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:09,652 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 40a49f11b38d4dcf87eb05631338c71a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732539487791 2024-11-25T12:58:09,652 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting b73d79d4de1c4c7f9057aaf4adf92424, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732539489080 2024-11-25T12:58:09,653 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/45b4db3b2f954395ac6a5ad67fcbed6e, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/a4e1a73720dd40e8a3fe8d17add0b9ee, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/48f0f15698d041a0b35a39bcdfae5ae0, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/3ba02a1801164515b68ae50df492f529] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp, totalSize=129.4 K 2024-11-25T12:58:09,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:09,653 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:09,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:09,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:09,653 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. files: [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/45b4db3b2f954395ac6a5ad67fcbed6e, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/a4e1a73720dd40e8a3fe8d17add0b9ee, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/48f0f15698d041a0b35a39bcdfae5ae0, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/3ba02a1801164515b68ae50df492f529] 2024-11-25T12:58:09,654 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45b4db3b2f954395ac6a5ad67fcbed6e, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732539485331 2024-11-25T12:58:09,655 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4e1a73720dd40e8a3fe8d17add0b9ee, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732539486575 2024-11-25T12:58:09,656 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48f0f15698d041a0b35a39bcdfae5ae0, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732539487791 2024-11-25T12:58:09,657 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ba02a1801164515b68ae50df492f529, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732539489080 2024-11-25T12:58:09,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:09,664 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:58:09,664 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9806b0fa8f18a18b9b918dba8c42bc67 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-25T12:58:09,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-25T12:58:09,675 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-25T12:58:09,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:09,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. as already flushing 2024-11-25T12:58:09,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:09,679 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:09,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:09,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:09,689 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9806b0fa8f18a18b9b918dba8c42bc67#B#compaction#114 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:58:09,697 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/B/d6f9df5c04104e5b9b4016160c7f9432 is 50, key is test_row_0/B:col10/1732539489240/Put/seqid=0 2024-11-25T12:58:09,728 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9806b0fa8f18a18b9b918dba8c42bc67] 2024-11-25T12:58:09,758 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125de7f2ff4f38749c19c92ec8b4c0e3ba5_9806b0fa8f18a18b9b918dba8c42bc67 is 50, key is test_row_0/A:col10/1732539489290/Put/seqid=0 2024-11-25T12:58:09,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:09,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59944 deadline: 1732539549739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:09,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:09,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60008 deadline: 1732539549750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:09,763 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:09,763 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:09,763 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:09,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:09,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:09,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:09,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:09,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59962 deadline: 1732539549759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:09,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:09,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59968 deadline: 1732539549768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:09,769 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:09,770 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:09,770 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:09,776 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241125b57c0338851841aa96acbad5ce886e16_9806b0fa8f18a18b9b918dba8c42bc67 store=[table=TestAcidGuarantees family=A region=9806b0fa8f18a18b9b918dba8c42bc67] 2024-11-25T12:58:09,779 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:09,779 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:09,779 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:09,782 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241125b57c0338851841aa96acbad5ce886e16_9806b0fa8f18a18b9b918dba8c42bc67, store=[table=TestAcidGuarantees family=A region=9806b0fa8f18a18b9b918dba8c42bc67] 2024-11-25T12:58:09,782 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125b57c0338851841aa96acbad5ce886e16_9806b0fa8f18a18b9b918dba8c42bc67 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9806b0fa8f18a18b9b918dba8c42bc67] 2024-11-25T12:58:09,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741961_1137 (size=12241) 2024-11-25T12:58:09,836 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-25T12:58:09,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:09,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. as already flushing 2024-11-25T12:58:09,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:09,837 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:09,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:09,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:09,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741962_1138 (size=14694) 2024-11-25T12:58:09,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741963_1139 (size=4469) 2024-11-25T12:58:09,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:09,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59944 deadline: 1732539549870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:09,873 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:09,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:09,873 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:09,873 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:09,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60008 deadline: 1732539549870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:09,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:09,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:09,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:09,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:09,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59968 deadline: 1732539549879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:09,885 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:09,885 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:09,885 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:09,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:09,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59962 deadline: 1732539549890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:09,894 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:09,894 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:09,894 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:09,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:09,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59966 deadline: 1732539549961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:09,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:09,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:09,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:09,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-25T12:58:09,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-25T12:58:09,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:09,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. as already flushing 2024-11-25T12:58:09,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:09,990 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:09,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:09,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:10,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:10,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59944 deadline: 1732539550080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:10,088 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:10,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:10,088 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:10,088 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:10,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60008 deadline: 1732539550080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:10,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:10,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:10,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:10,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:10,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59968 deadline: 1732539550090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:10,097 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:10,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:10,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:10,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:10,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59962 deadline: 1732539550107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:10,110 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:10,110 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:10,110 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:10,145 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-25T12:58:10,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:10,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. as already flushing 2024-11-25T12:58:10,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:10,147 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:10,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:10,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:10,213 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/B/d6f9df5c04104e5b9b4016160c7f9432 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/d6f9df5c04104e5b9b4016160c7f9432 2024-11-25T12:58:10,228 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9806b0fa8f18a18b9b918dba8c42bc67/B of 9806b0fa8f18a18b9b918dba8c42bc67 into d6f9df5c04104e5b9b4016160c7f9432(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:58:10,228 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9806b0fa8f18a18b9b918dba8c42bc67: 2024-11-25T12:58:10,228 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., storeName=9806b0fa8f18a18b9b918dba8c42bc67/B, priority=12, startTime=1732539489645; duration=0sec 2024-11-25T12:58:10,229 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:58:10,229 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9806b0fa8f18a18b9b918dba8c42bc67:B 2024-11-25T12:58:10,229 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:58:10,241 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:58:10,241 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): 9806b0fa8f18a18b9b918dba8c42bc67/C is initiating minor compaction (all files) 2024-11-25T12:58:10,241 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9806b0fa8f18a18b9b918dba8c42bc67/C in TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:10,241 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/d240fdce83f94459be2a8f384cf3dc9d, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/19973b88a14e4e17ab77d2f88fd60538, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/35804b79e49146b58ebfcd1146434bb5, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/046ce0ad39ae48eaa1206d3508685b7e] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp, totalSize=47.0 K 2024-11-25T12:58:10,243 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting d240fdce83f94459be2a8f384cf3dc9d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732539485331 2024-11-25T12:58:10,243 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 19973b88a14e4e17ab77d2f88fd60538, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732539486575 2024-11-25T12:58:10,244 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 35804b79e49146b58ebfcd1146434bb5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732539487791 2024-11-25T12:58:10,244 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 046ce0ad39ae48eaa1206d3508685b7e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732539489080 2024-11-25T12:58:10,247 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:10,252 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125de7f2ff4f38749c19c92ec8b4c0e3ba5_9806b0fa8f18a18b9b918dba8c42bc67 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125de7f2ff4f38749c19c92ec8b4c0e3ba5_9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:10,254 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/708de6d9542746ef9414786c61c26e05, store: [table=TestAcidGuarantees family=A region=9806b0fa8f18a18b9b918dba8c42bc67] 2024-11-25T12:58:10,254 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/708de6d9542746ef9414786c61c26e05 is 175, key is test_row_0/A:col10/1732539489290/Put/seqid=0 2024-11-25T12:58:10,261 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9806b0fa8f18a18b9b918dba8c42bc67#A#compaction#115 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:58:10,261 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/d3eee17f68454bc2814e07605f204184 is 175, key is test_row_0/A:col10/1732539489240/Put/seqid=0 2024-11-25T12:58:10,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741964_1140 (size=39649) 2024-11-25T12:58:10,301 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-25T12:58:10,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:10,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. as already flushing 2024-11-25T12:58:10,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:10,303 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:10,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:10,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:10,316 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9806b0fa8f18a18b9b918dba8c42bc67#C#compaction#117 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:58:10,317 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/C/d5674cb220384b95b9be680e38c1f2ba is 50, key is test_row_0/C:col10/1732539489240/Put/seqid=0 2024-11-25T12:58:10,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741965_1141 (size=31195) 2024-11-25T12:58:10,361 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/d3eee17f68454bc2814e07605f204184 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/d3eee17f68454bc2814e07605f204184 2024-11-25T12:58:10,376 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9806b0fa8f18a18b9b918dba8c42bc67/A of 9806b0fa8f18a18b9b918dba8c42bc67 into d3eee17f68454bc2814e07605f204184(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:58:10,376 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9806b0fa8f18a18b9b918dba8c42bc67: 2024-11-25T12:58:10,376 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., storeName=9806b0fa8f18a18b9b918dba8c42bc67/A, priority=12, startTime=1732539489645; duration=0sec 2024-11-25T12:58:10,376 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:58:10,376 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9806b0fa8f18a18b9b918dba8c42bc67:A 2024-11-25T12:58:10,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741966_1142 (size=12241) 2024-11-25T12:58:10,396 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/C/d5674cb220384b95b9be680e38c1f2ba as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/d5674cb220384b95b9be680e38c1f2ba 2024-11-25T12:58:10,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:10,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60008 deadline: 1732539550400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:10,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:10,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:10,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:10,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59944 deadline: 1732539550400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:10,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:10,403 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:10,403 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:10,403 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:10,414 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9806b0fa8f18a18b9b918dba8c42bc67/C of 9806b0fa8f18a18b9b918dba8c42bc67 into d5674cb220384b95b9be680e38c1f2ba(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:58:10,414 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9806b0fa8f18a18b9b918dba8c42bc67: 2024-11-25T12:58:10,414 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., storeName=9806b0fa8f18a18b9b918dba8c42bc67/C, priority=12, startTime=1732539489645; duration=0sec 2024-11-25T12:58:10,414 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:58:10,414 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9806b0fa8f18a18b9b918dba8c42bc67:C 2024-11-25T12:58:10,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:10,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59968 deadline: 1732539550411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:10,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:10,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:10,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:10,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:10,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59962 deadline: 1732539550420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:10,424 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:10,424 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:10,424 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:10,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-25T12:58:10,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:10,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. as already flushing 2024-11-25T12:58:10,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:10,462 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:10,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:10,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:10,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:10,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59966 deadline: 1732539550469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:10,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:10,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:10,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:10,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-25T12:58:10,616 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-25T12:58:10,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:10,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. as already flushing 2024-11-25T12:58:10,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:10,617 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:10,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:10,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:10,690 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=131, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/708de6d9542746ef9414786c61c26e05 2024-11-25T12:58:10,705 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/B/5f22d2daace04d9c9e56783b4b996c0e is 50, key is test_row_0/B:col10/1732539489290/Put/seqid=0 2024-11-25T12:58:10,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741967_1143 (size=12101) 2024-11-25T12:58:10,742 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/B/5f22d2daace04d9c9e56783b4b996c0e 2024-11-25T12:58:10,761 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/C/2d4a1f40c93c4ca59c478dff3b8b0776 is 50, key is test_row_0/C:col10/1732539489290/Put/seqid=0 2024-11-25T12:58:10,771 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-25T12:58:10,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:10,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. as already flushing 2024-11-25T12:58:10,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:10,772 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:10,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:10,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:10,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741968_1144 (size=12101) 2024-11-25T12:58:10,810 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/C/2d4a1f40c93c4ca59c478dff3b8b0776 2024-11-25T12:58:10,825 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/708de6d9542746ef9414786c61c26e05 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/708de6d9542746ef9414786c61c26e05 2024-11-25T12:58:10,831 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/708de6d9542746ef9414786c61c26e05, entries=200, sequenceid=131, filesize=38.7 K 2024-11-25T12:58:10,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/B/5f22d2daace04d9c9e56783b4b996c0e as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/5f22d2daace04d9c9e56783b4b996c0e 2024-11-25T12:58:10,839 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/5f22d2daace04d9c9e56783b4b996c0e, entries=150, sequenceid=131, filesize=11.8 K 2024-11-25T12:58:10,841 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/C/2d4a1f40c93c4ca59c478dff3b8b0776 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/2d4a1f40c93c4ca59c478dff3b8b0776 2024-11-25T12:58:10,852 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/2d4a1f40c93c4ca59c478dff3b8b0776, entries=150, sequenceid=131, filesize=11.8 K 2024-11-25T12:58:10,853 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 9806b0fa8f18a18b9b918dba8c42bc67 in 1189ms, sequenceid=131, compaction requested=false 2024-11-25T12:58:10,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9806b0fa8f18a18b9b918dba8c42bc67: 2024-11-25T12:58:10,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:10,914 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:58:10,914 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9806b0fa8f18a18b9b918dba8c42bc67 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-25T12:58:10,928 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-25T12:58:10,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:10,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. as already flushing 2024-11-25T12:58:10,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:10,928 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:10,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:10,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:10,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:10,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60008 deadline: 1732539550942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:10,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:10,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:10,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:10,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:10,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:10,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59944 deadline: 1732539550957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:10,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59962 deadline: 1732539550957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:10,961 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:10,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:10,961 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:10,962 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:10,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59968 deadline: 1732539550960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:10,965 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:10,965 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:10,965 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:10,965 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:10,966 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:10,966 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:10,966 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112518b243159d494ae983b9012acf8fb9bb_9806b0fa8f18a18b9b918dba8c42bc67 is 50, key is test_row_0/A:col10/1732539489718/Put/seqid=0 2024-11-25T12:58:11,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741969_1145 (size=14794) 2024-11-25T12:58:11,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:11,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60008 deadline: 1732539551060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:11,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:11,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:11,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:11,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:11,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59968 deadline: 1732539551070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:11,076 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:11,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:11,077 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:11,077 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:11,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59962 deadline: 1732539551070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:11,077 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:11,077 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:11,077 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:11,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:11,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59944 deadline: 1732539551071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:11,077 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:11,077 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:11,077 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:11,083 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-25T12:58:11,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:11,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. as already flushing 2024-11-25T12:58:11,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:11,083 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:11,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:11,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:11,236 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-25T12:58:11,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:11,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. as already flushing 2024-11-25T12:58:11,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:11,237 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:11,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:11,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:11,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:11,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60008 deadline: 1732539551275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:11,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:11,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:11,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:11,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:11,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59968 deadline: 1732539551280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:11,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:11,283 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:11,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59962 deadline: 1732539551280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:11,283 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:11,283 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:11,283 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:11,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:11,283 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:11,283 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:11,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59944 deadline: 1732539551281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:11,284 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:11,284 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:11,284 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:11,390 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-25T12:58:11,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:11,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. as already flushing 2024-11-25T12:58:11,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:11,390 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:11,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:11,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:11,409 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:11,439 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112518b243159d494ae983b9012acf8fb9bb_9806b0fa8f18a18b9b918dba8c42bc67 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112518b243159d494ae983b9012acf8fb9bb_9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:11,452 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/b9d70a8295d946d38fdd8d9815fa2fee, store: [table=TestAcidGuarantees family=A region=9806b0fa8f18a18b9b918dba8c42bc67] 2024-11-25T12:58:11,453 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/b9d70a8295d946d38fdd8d9815fa2fee is 175, key is test_row_0/A:col10/1732539489718/Put/seqid=0 2024-11-25T12:58:11,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741970_1146 (size=39749) 2024-11-25T12:58:11,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:11,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59966 deadline: 1732539551491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:11,497 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 1ff8009e463e,35905,1732539449246 for 'test_row_2' in 9806b0fa8f18a18b9b918dba8c42bc67 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T12:58:11,498 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:11,498 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:11,498 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:11,507 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=161, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/b9d70a8295d946d38fdd8d9815fa2fee 2024-11-25T12:58:11,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-25T12:58:11,540 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/B/9977f799cb0a4fd68d5f97914d5334be is 50, key is test_row_0/B:col10/1732539489718/Put/seqid=0 2024-11-25T12:58:11,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-25T12:58:11,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:11,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. as already flushing 2024-11-25T12:58:11,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:11,547 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:11,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:11,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:11,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741971_1147 (size=12151) 2024-11-25T12:58:11,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:11,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:11,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59968 deadline: 1732539551591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:11,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60008 deadline: 1732539551590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:11,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:11,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:11,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:11,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:11,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59944 deadline: 1732539551591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:11,594 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:11,594 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:11,594 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:11,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:11,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:11,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:11,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:11,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59962 deadline: 1732539551591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:11,595 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:11,595 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:11,595 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:11,700 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-25T12:58:11,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:11,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. as already flushing 2024-11-25T12:58:11,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:11,700 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:11,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:11,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:11,863 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-25T12:58:11,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:11,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. as already flushing 2024-11-25T12:58:11,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:11,864 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:11,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:11,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:11,991 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/B/9977f799cb0a4fd68d5f97914d5334be 2024-11-25T12:58:12,019 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-25T12:58:12,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:12,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. as already flushing 2024-11-25T12:58:12,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:12,019 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:12,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:12,020 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/C/5b8afe89241a411e8ba40e16f77ecc66 is 50, key is test_row_0/C:col10/1732539489718/Put/seqid=0 2024-11-25T12:58:12,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:12,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741972_1148 (size=12151) 2024-11-25T12:58:12,062 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/C/5b8afe89241a411e8ba40e16f77ecc66 2024-11-25T12:58:12,072 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/b9d70a8295d946d38fdd8d9815fa2fee as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/b9d70a8295d946d38fdd8d9815fa2fee 2024-11-25T12:58:12,079 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/b9d70a8295d946d38fdd8d9815fa2fee, entries=200, sequenceid=161, filesize=38.8 K 2024-11-25T12:58:12,080 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/B/9977f799cb0a4fd68d5f97914d5334be as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/9977f799cb0a4fd68d5f97914d5334be 2024-11-25T12:58:12,097 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/9977f799cb0a4fd68d5f97914d5334be, entries=150, sequenceid=161, filesize=11.9 K 2024-11-25T12:58:12,098 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/C/5b8afe89241a411e8ba40e16f77ecc66 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/5b8afe89241a411e8ba40e16f77ecc66 2024-11-25T12:58:12,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:12,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60008 deadline: 1732539552100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:12,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:12,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:12,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:12,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:12,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59968 deadline: 1732539552100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:12,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:12,105 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:12,105 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:12,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59944 deadline: 1732539552100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:12,105 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:12,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:12,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59962 deadline: 1732539552100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:12,105 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:12,105 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:12,105 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:12,106 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:12,106 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:12,106 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:12,107 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/5b8afe89241a411e8ba40e16f77ecc66, entries=150, sequenceid=161, filesize=11.9 K 2024-11-25T12:58:12,108 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=46.96 KB/48090 for 9806b0fa8f18a18b9b918dba8c42bc67 in 1194ms, sequenceid=161, compaction requested=true 2024-11-25T12:58:12,108 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9806b0fa8f18a18b9b918dba8c42bc67: 2024-11-25T12:58:12,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9806b0fa8f18a18b9b918dba8c42bc67:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:58:12,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:58:12,108 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:58:12,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9806b0fa8f18a18b9b918dba8c42bc67:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:58:12,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:58:12,108 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:58:12,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9806b0fa8f18a18b9b918dba8c42bc67:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:58:12,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:58:12,109 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110593 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:58:12,109 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1541): 9806b0fa8f18a18b9b918dba8c42bc67/A is initiating minor compaction (all files) 2024-11-25T12:58:12,109 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9806b0fa8f18a18b9b918dba8c42bc67/A in TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:12,110 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/d3eee17f68454bc2814e07605f204184, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/708de6d9542746ef9414786c61c26e05, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/b9d70a8295d946d38fdd8d9815fa2fee] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp, totalSize=108.0 K 2024-11-25T12:58:12,110 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:12,110 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. files: [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/d3eee17f68454bc2814e07605f204184, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/708de6d9542746ef9414786c61c26e05, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/b9d70a8295d946d38fdd8d9815fa2fee] 2024-11-25T12:58:12,110 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36493 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:58:12,110 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): 9806b0fa8f18a18b9b918dba8c42bc67/B is initiating minor compaction (all files) 2024-11-25T12:58:12,110 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9806b0fa8f18a18b9b918dba8c42bc67/B in TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:12,110 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/d6f9df5c04104e5b9b4016160c7f9432, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/5f22d2daace04d9c9e56783b4b996c0e, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/9977f799cb0a4fd68d5f97914d5334be] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp, totalSize=35.6 K 2024-11-25T12:58:12,111 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting d6f9df5c04104e5b9b4016160c7f9432, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732539489080 2024-11-25T12:58:12,111 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3eee17f68454bc2814e07605f204184, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732539489080 2024-11-25T12:58:12,111 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f22d2daace04d9c9e56783b4b996c0e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732539489287 2024-11-25T12:58:12,111 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 708de6d9542746ef9414786c61c26e05, keycount=200, bloomtype=ROW, size=38.7 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732539489272 2024-11-25T12:58:12,112 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 9977f799cb0a4fd68d5f97914d5334be, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1732539489718 2024-11-25T12:58:12,112 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9d70a8295d946d38fdd8d9815fa2fee, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1732539489718 2024-11-25T12:58:12,136 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9806b0fa8f18a18b9b918dba8c42bc67] 2024-11-25T12:58:12,146 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9806b0fa8f18a18b9b918dba8c42bc67#B#compaction#124 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:58:12,146 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/B/cec2ac5785114183babe8dda33cabcb8 is 50, key is test_row_0/B:col10/1732539489718/Put/seqid=0 2024-11-25T12:58:12,156 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112537b242fa65cc4f899c4d02314a325e65_9806b0fa8f18a18b9b918dba8c42bc67 store=[table=TestAcidGuarantees family=A region=9806b0fa8f18a18b9b918dba8c42bc67] 2024-11-25T12:58:12,160 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112537b242fa65cc4f899c4d02314a325e65_9806b0fa8f18a18b9b918dba8c42bc67, store=[table=TestAcidGuarantees family=A region=9806b0fa8f18a18b9b918dba8c42bc67] 2024-11-25T12:58:12,161 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112537b242fa65cc4f899c4d02314a325e65_9806b0fa8f18a18b9b918dba8c42bc67 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9806b0fa8f18a18b9b918dba8c42bc67] 2024-11-25T12:58:12,174 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-25T12:58:12,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:12,175 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 9806b0fa8f18a18b9b918dba8c42bc67 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-25T12:58:12,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741973_1149 (size=12493) 2024-11-25T12:58:12,211 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/B/cec2ac5785114183babe8dda33cabcb8 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/cec2ac5785114183babe8dda33cabcb8 2024-11-25T12:58:12,231 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9806b0fa8f18a18b9b918dba8c42bc67/B of 9806b0fa8f18a18b9b918dba8c42bc67 into cec2ac5785114183babe8dda33cabcb8(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:58:12,231 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9806b0fa8f18a18b9b918dba8c42bc67: 2024-11-25T12:58:12,232 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., storeName=9806b0fa8f18a18b9b918dba8c42bc67/B, priority=13, startTime=1732539492108; duration=0sec 2024-11-25T12:58:12,232 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:58:12,232 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9806b0fa8f18a18b9b918dba8c42bc67:B 2024-11-25T12:58:12,232 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:58:12,235 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36493 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:58:12,235 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): 9806b0fa8f18a18b9b918dba8c42bc67/C is initiating minor compaction (all files) 2024-11-25T12:58:12,236 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9806b0fa8f18a18b9b918dba8c42bc67/C in TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:12,236 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/d5674cb220384b95b9be680e38c1f2ba, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/2d4a1f40c93c4ca59c478dff3b8b0776, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/5b8afe89241a411e8ba40e16f77ecc66] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp, totalSize=35.6 K 2024-11-25T12:58:12,236 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting d5674cb220384b95b9be680e38c1f2ba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732539489080 2024-11-25T12:58:12,237 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d4a1f40c93c4ca59c478dff3b8b0776, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732539489287 2024-11-25T12:58:12,239 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b8afe89241a411e8ba40e16f77ecc66, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1732539489718 2024-11-25T12:58:12,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741974_1150 (size=4469) 2024-11-25T12:58:12,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411254e6c47e9a4be405891be7b6b96c0b272_9806b0fa8f18a18b9b918dba8c42bc67 is 50, key is test_row_0/A:col10/1732539490948/Put/seqid=0 2024-11-25T12:58:12,269 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9806b0fa8f18a18b9b918dba8c42bc67#C#compaction#126 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:58:12,270 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/C/e113ca4f6454470892132618e44b756e is 50, key is test_row_0/C:col10/1732539489718/Put/seqid=0 2024-11-25T12:58:12,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741975_1151 (size=12304) 2024-11-25T12:58:12,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741976_1152 (size=12493) 2024-11-25T12:58:12,325 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/C/e113ca4f6454470892132618e44b756e as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/e113ca4f6454470892132618e44b756e 2024-11-25T12:58:12,333 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9806b0fa8f18a18b9b918dba8c42bc67/C of 9806b0fa8f18a18b9b918dba8c42bc67 into e113ca4f6454470892132618e44b756e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:58:12,333 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9806b0fa8f18a18b9b918dba8c42bc67: 2024-11-25T12:58:12,333 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., storeName=9806b0fa8f18a18b9b918dba8c42bc67/C, priority=13, startTime=1732539492108; duration=0sec 2024-11-25T12:58:12,333 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:58:12,333 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9806b0fa8f18a18b9b918dba8c42bc67:C 2024-11-25T12:58:12,655 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9806b0fa8f18a18b9b918dba8c42bc67#A#compaction#123 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:58:12,655 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/2da1382ddb824217880c1e6f67c54369 is 175, key is test_row_0/A:col10/1732539489718/Put/seqid=0 2024-11-25T12:58:12,712 INFO [master/1ff8009e463e:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-25T12:58:12,712 INFO [master/1ff8009e463e:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-25T12:58:12,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:12,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741977_1153 (size=31447) 2024-11-25T12:58:12,749 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411254e6c47e9a4be405891be7b6b96c0b272_9806b0fa8f18a18b9b918dba8c42bc67 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411254e6c47e9a4be405891be7b6b96c0b272_9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:12,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/09702298e36948539ba0a20f52945123, store: [table=TestAcidGuarantees family=A region=9806b0fa8f18a18b9b918dba8c42bc67] 2024-11-25T12:58:12,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/09702298e36948539ba0a20f52945123 is 175, key is test_row_0/A:col10/1732539490948/Put/seqid=0 2024-11-25T12:58:12,765 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/2da1382ddb824217880c1e6f67c54369 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/2da1382ddb824217880c1e6f67c54369 2024-11-25T12:58:12,780 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9806b0fa8f18a18b9b918dba8c42bc67/A of 9806b0fa8f18a18b9b918dba8c42bc67 into 2da1382ddb824217880c1e6f67c54369(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:58:12,780 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9806b0fa8f18a18b9b918dba8c42bc67: 2024-11-25T12:58:12,780 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., storeName=9806b0fa8f18a18b9b918dba8c42bc67/A, priority=13, startTime=1732539492108; duration=0sec 2024-11-25T12:58:12,780 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:58:12,780 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9806b0fa8f18a18b9b918dba8c42bc67:A 2024-11-25T12:58:12,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741978_1154 (size=31105) 2024-11-25T12:58:12,801 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=171, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/09702298e36948539ba0a20f52945123 2024-11-25T12:58:12,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/B/f1ba489c109f49b382b0f6d0347469c5 is 50, key is test_row_0/B:col10/1732539490948/Put/seqid=0 2024-11-25T12:58:12,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741979_1155 (size=12151) 2024-11-25T12:58:13,113 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. as already flushing 2024-11-25T12:58:13,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:13,270 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/B/f1ba489c109f49b382b0f6d0347469c5 2024-11-25T12:58:13,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:13,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59944 deadline: 1732539553258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:13,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:13,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59968 deadline: 1732539553263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:13,272 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:13,272 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:13,272 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:13,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:13,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60008 deadline: 1732539553264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:13,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:13,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:13,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:13,275 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:13,275 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:13,275 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:13,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/C/3ba99245a9484ed3845e735e31e9efa5 is 50, key is test_row_0/C:col10/1732539490948/Put/seqid=0 2024-11-25T12:58:13,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:13,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59962 deadline: 1732539553272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:13,292 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:13,292 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:13,292 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:13,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741980_1156 (size=12151) 2024-11-25T12:58:13,361 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/C/3ba99245a9484ed3845e735e31e9efa5 2024-11-25T12:58:13,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/09702298e36948539ba0a20f52945123 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/09702298e36948539ba0a20f52945123 2024-11-25T12:58:13,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:13,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:13,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:13,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59944 deadline: 1732539553380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:13,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59968 deadline: 1732539553380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:13,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:13,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:13,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:13,402 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:13,402 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:13,402 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:13,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60008 deadline: 1732539553380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:13,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:13,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:13,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:13,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:13,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59962 deadline: 1732539553401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:13,405 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:13,405 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:13,406 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:13,408 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/09702298e36948539ba0a20f52945123, entries=150, sequenceid=171, filesize=30.4 K 2024-11-25T12:58:13,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/B/f1ba489c109f49b382b0f6d0347469c5 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/f1ba489c109f49b382b0f6d0347469c5 2024-11-25T12:58:13,420 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/f1ba489c109f49b382b0f6d0347469c5, entries=150, sequenceid=171, filesize=11.9 K 2024-11-25T12:58:13,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/C/3ba99245a9484ed3845e735e31e9efa5 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/3ba99245a9484ed3845e735e31e9efa5 2024-11-25T12:58:13,430 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/3ba99245a9484ed3845e735e31e9efa5, entries=150, sequenceid=171, filesize=11.9 K 2024-11-25T12:58:13,432 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 9806b0fa8f18a18b9b918dba8c42bc67 in 1258ms, sequenceid=171, compaction requested=false 2024-11-25T12:58:13,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 9806b0fa8f18a18b9b918dba8c42bc67: 2024-11-25T12:58:13,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:13,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-25T12:58:13,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-25T12:58:13,446 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-25T12:58:13,446 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.0950 sec 2024-11-25T12:58:13,448 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 4.1040 sec 2024-11-25T12:58:13,514 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:58:13,514 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9806b0fa8f18a18b9b918dba8c42bc67 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-25T12:58:13,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:13,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-25T12:58:13,540 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:58:13,541 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:58:13,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-25T12:58:13,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-25T12:58:13,543 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:58:13,543 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411252d4828d17d6b4d0d99526ee70a6f92cf_9806b0fa8f18a18b9b918dba8c42bc67 is 50, key is test_row_0/A:col10/1732539493263/Put/seqid=0 2024-11-25T12:58:13,544 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:58:13,544 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:58:13,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:13,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59966 deadline: 1732539553559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:13,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:13,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:13,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:13,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741981_1157 (size=12304) 2024-11-25T12:58:13,593 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:13,602 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411252d4828d17d6b4d0d99526ee70a6f92cf_9806b0fa8f18a18b9b918dba8c42bc67 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411252d4828d17d6b4d0d99526ee70a6f92cf_9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:13,603 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/ed64e7db165e42c9a08986295b673456, store: [table=TestAcidGuarantees family=A region=9806b0fa8f18a18b9b918dba8c42bc67] 2024-11-25T12:58:13,604 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/ed64e7db165e42c9a08986295b673456 is 175, key is test_row_0/A:col10/1732539493263/Put/seqid=0 2024-11-25T12:58:13,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741982_1158 (size=31105) 2024-11-25T12:58:13,613 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=201, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/ed64e7db165e42c9a08986295b673456 2024-11-25T12:58:13,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:13,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59944 deadline: 1732539553610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:13,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:13,615 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:13,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59968 deadline: 1732539553610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:13,615 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:13,615 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:13,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:13,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:13,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:13,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60008 deadline: 1732539553610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:13,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:13,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:13,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:13,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:13,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:13,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59962 deadline: 1732539553611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:13,616 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:13,616 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:13,616 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:13,629 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/B/79300711dbf144e29559ec96e5759daa is 50, key is test_row_0/B:col10/1732539493263/Put/seqid=0 2024-11-25T12:58:13,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741983_1159 (size=12151) 2024-11-25T12:58:13,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-25T12:58:13,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:13,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59966 deadline: 1732539553670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:13,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:13,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:13,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:13,696 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-25T12:58:13,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:13,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. as already flushing 2024-11-25T12:58:13,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:13,697 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:13,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:13,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:13,849 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-25T12:58:13,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:13,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. as already flushing 2024-11-25T12:58:13,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:13,850 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:13,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:13,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:13,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-25T12:58:13,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:13,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59966 deadline: 1732539553879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:13,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:13,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:13,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:13,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:13,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59944 deadline: 1732539553921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:13,924 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:13,924 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:13,924 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:13,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:13,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59968 deadline: 1732539553922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:13,928 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:13,928 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:13,928 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:13,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:13,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60008 deadline: 1732539553922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:13,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:13,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:13,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:13,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:13,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59962 deadline: 1732539553931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:13,937 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:13,937 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:13,937 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:14,004 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-25T12:58:14,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:14,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. as already flushing 2024-11-25T12:58:14,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:14,007 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:14,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:14,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:14,048 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/B/79300711dbf144e29559ec96e5759daa 2024-11-25T12:58:14,089 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/C/86fa443247b14e229df3cecdf466d69d is 50, key is test_row_0/C:col10/1732539493263/Put/seqid=0 2024-11-25T12:58:14,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741984_1160 (size=12151) 2024-11-25T12:58:14,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-25T12:58:14,172 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-25T12:58:14,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:14,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. as already flushing 2024-11-25T12:58:14,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:14,172 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:14,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:14,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:14,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:14,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59966 deadline: 1732539554190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:14,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:14,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:14,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:14,329 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-25T12:58:14,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:14,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. as already flushing 2024-11-25T12:58:14,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:14,330 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:14,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:14,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:14,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:14,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59944 deadline: 1732539554431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:14,438 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:14,438 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:14,438 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:14,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:14,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59968 deadline: 1732539554442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:14,444 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:14,444 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:14,444 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:14,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:14,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60008 deadline: 1732539554442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:14,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:14,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:14,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:14,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:14,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59962 deadline: 1732539554450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:14,460 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:14,460 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:14,460 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:14,483 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-25T12:58:14,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:14,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. as already flushing 2024-11-25T12:58:14,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:14,484 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:14,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:14,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:14,498 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/C/86fa443247b14e229df3cecdf466d69d 2024-11-25T12:58:14,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/ed64e7db165e42c9a08986295b673456 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/ed64e7db165e42c9a08986295b673456 2024-11-25T12:58:14,523 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/ed64e7db165e42c9a08986295b673456, entries=150, sequenceid=201, filesize=30.4 K 2024-11-25T12:58:14,527 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/B/79300711dbf144e29559ec96e5759daa as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/79300711dbf144e29559ec96e5759daa 2024-11-25T12:58:14,536 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/79300711dbf144e29559ec96e5759daa, entries=150, sequenceid=201, filesize=11.9 K 2024-11-25T12:58:14,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/C/86fa443247b14e229df3cecdf466d69d as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/86fa443247b14e229df3cecdf466d69d 2024-11-25T12:58:14,564 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/86fa443247b14e229df3cecdf466d69d, entries=150, sequenceid=201, filesize=11.9 K 2024-11-25T12:58:14,569 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 9806b0fa8f18a18b9b918dba8c42bc67 in 1055ms, sequenceid=201, compaction requested=true 2024-11-25T12:58:14,569 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9806b0fa8f18a18b9b918dba8c42bc67: 2024-11-25T12:58:14,570 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:58:14,571 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93657 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:58:14,571 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1541): 9806b0fa8f18a18b9b918dba8c42bc67/A is initiating minor compaction (all files) 2024-11-25T12:58:14,571 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9806b0fa8f18a18b9b918dba8c42bc67/A in TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:14,571 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/2da1382ddb824217880c1e6f67c54369, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/09702298e36948539ba0a20f52945123, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/ed64e7db165e42c9a08986295b673456] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp, totalSize=91.5 K 2024-11-25T12:58:14,571 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:14,571 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. files: [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/2da1382ddb824217880c1e6f67c54369, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/09702298e36948539ba0a20f52945123, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/ed64e7db165e42c9a08986295b673456] 2024-11-25T12:58:14,572 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2da1382ddb824217880c1e6f67c54369, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1732539489718 2024-11-25T12:58:14,573 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09702298e36948539ba0a20f52945123, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732539490945 2024-11-25T12:58:14,573 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed64e7db165e42c9a08986295b673456, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1732539493242 2024-11-25T12:58:14,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9806b0fa8f18a18b9b918dba8c42bc67:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:58:14,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:58:14,573 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:58:14,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9806b0fa8f18a18b9b918dba8c42bc67:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:58:14,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:58:14,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9806b0fa8f18a18b9b918dba8c42bc67:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:58:14,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:58:14,574 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:58:14,574 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): 9806b0fa8f18a18b9b918dba8c42bc67/B is initiating minor compaction (all files) 2024-11-25T12:58:14,574 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9806b0fa8f18a18b9b918dba8c42bc67/B in TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:14,574 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/cec2ac5785114183babe8dda33cabcb8, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/f1ba489c109f49b382b0f6d0347469c5, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/79300711dbf144e29559ec96e5759daa] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp, totalSize=35.9 K 2024-11-25T12:58:14,575 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting cec2ac5785114183babe8dda33cabcb8, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1732539489718 2024-11-25T12:58:14,576 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting f1ba489c109f49b382b0f6d0347469c5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732539490945 2024-11-25T12:58:14,576 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 79300711dbf144e29559ec96e5759daa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1732539493242 2024-11-25T12:58:14,582 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9806b0fa8f18a18b9b918dba8c42bc67] 2024-11-25T12:58:14,585 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241125a52996187c5b4a17bf76ce8def8000be_9806b0fa8f18a18b9b918dba8c42bc67 store=[table=TestAcidGuarantees family=A region=9806b0fa8f18a18b9b918dba8c42bc67] 2024-11-25T12:58:14,587 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241125a52996187c5b4a17bf76ce8def8000be_9806b0fa8f18a18b9b918dba8c42bc67, store=[table=TestAcidGuarantees family=A region=9806b0fa8f18a18b9b918dba8c42bc67] 2024-11-25T12:58:14,587 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125a52996187c5b4a17bf76ce8def8000be_9806b0fa8f18a18b9b918dba8c42bc67 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9806b0fa8f18a18b9b918dba8c42bc67] 2024-11-25T12:58:14,596 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9806b0fa8f18a18b9b918dba8c42bc67#B#compaction#133 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:58:14,596 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/B/daf2b242c2564b7380145d0d5553c1e1 is 50, key is test_row_0/B:col10/1732539493263/Put/seqid=0 2024-11-25T12:58:14,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741985_1161 (size=4469) 2024-11-25T12:58:14,634 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9806b0fa8f18a18b9b918dba8c42bc67#A#compaction#132 average throughput is 0.47 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:58:14,635 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/620ace6715e54c00abfa1de3f1407cf8 is 175, key is test_row_0/A:col10/1732539493263/Put/seqid=0 2024-11-25T12:58:14,650 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-25T12:58:14,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:14,650 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 9806b0fa8f18a18b9b918dba8c42bc67 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-25T12:58:14,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741986_1162 (size=12595) 2024-11-25T12:58:14,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741987_1163 (size=31549) 2024-11-25T12:58:14,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125f25c3593df774cedb157b5c75b80c83d_9806b0fa8f18a18b9b918dba8c42bc67 is 50, key is test_row_0/A:col10/1732539493547/Put/seqid=0 2024-11-25T12:58:14,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-25T12:58:14,687 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/620ace6715e54c00abfa1de3f1407cf8 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/620ace6715e54c00abfa1de3f1407cf8 2024-11-25T12:58:14,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741988_1164 (size=12304) 2024-11-25T12:58:14,698 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9806b0fa8f18a18b9b918dba8c42bc67/A of 9806b0fa8f18a18b9b918dba8c42bc67 into 620ace6715e54c00abfa1de3f1407cf8(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:58:14,698 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9806b0fa8f18a18b9b918dba8c42bc67: 2024-11-25T12:58:14,698 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., storeName=9806b0fa8f18a18b9b918dba8c42bc67/A, priority=13, startTime=1732539494569; duration=0sec 2024-11-25T12:58:14,699 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:58:14,699 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9806b0fa8f18a18b9b918dba8c42bc67:A 2024-11-25T12:58:14,699 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:58:14,700 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:58:14,700 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1541): 9806b0fa8f18a18b9b918dba8c42bc67/C is initiating minor compaction (all files) 2024-11-25T12:58:14,700 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9806b0fa8f18a18b9b918dba8c42bc67/C in TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:14,701 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/e113ca4f6454470892132618e44b756e, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/3ba99245a9484ed3845e735e31e9efa5, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/86fa443247b14e229df3cecdf466d69d] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp, totalSize=35.9 K 2024-11-25T12:58:14,701 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting e113ca4f6454470892132618e44b756e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1732539489718 2024-11-25T12:58:14,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:14,704 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ba99245a9484ed3845e735e31e9efa5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732539490945 2024-11-25T12:58:14,704 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. as already flushing 2024-11-25T12:58:14,704 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86fa443247b14e229df3cecdf466d69d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1732539493242 2024-11-25T12:58:14,717 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9806b0fa8f18a18b9b918dba8c42bc67#C#compaction#135 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:58:14,717 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/C/43140d42e3454be4aaee229c3ed44be5 is 50, key is test_row_0/C:col10/1732539493263/Put/seqid=0 2024-11-25T12:58:14,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741989_1165 (size=12595) 2024-11-25T12:58:14,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:14,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59966 deadline: 1732539554807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:14,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:14,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:14,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:14,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:14,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59966 deadline: 1732539554910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:14,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:14,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:14,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:15,066 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/B/daf2b242c2564b7380145d0d5553c1e1 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/daf2b242c2564b7380145d0d5553c1e1 2024-11-25T12:58:15,073 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9806b0fa8f18a18b9b918dba8c42bc67/B of 9806b0fa8f18a18b9b918dba8c42bc67 into daf2b242c2564b7380145d0d5553c1e1(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:58:15,073 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9806b0fa8f18a18b9b918dba8c42bc67: 2024-11-25T12:58:15,073 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., storeName=9806b0fa8f18a18b9b918dba8c42bc67/B, priority=13, startTime=1732539494573; duration=0sec 2024-11-25T12:58:15,073 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:58:15,073 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9806b0fa8f18a18b9b918dba8c42bc67:B 2024-11-25T12:58:15,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:15,088 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125f25c3593df774cedb157b5c75b80c83d_9806b0fa8f18a18b9b918dba8c42bc67 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125f25c3593df774cedb157b5c75b80c83d_9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:15,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/ef98c514cf7d408aad8ca10600c819a7, store: [table=TestAcidGuarantees family=A region=9806b0fa8f18a18b9b918dba8c42bc67] 2024-11-25T12:58:15,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/ef98c514cf7d408aad8ca10600c819a7 is 175, key is test_row_0/A:col10/1732539493547/Put/seqid=0 2024-11-25T12:58:15,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741990_1166 (size=31105) 2024-11-25T12:58:15,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:15,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59966 deadline: 1732539555120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:15,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:15,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:15,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:15,136 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/C/43140d42e3454be4aaee229c3ed44be5 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/43140d42e3454be4aaee229c3ed44be5 2024-11-25T12:58:15,146 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9806b0fa8f18a18b9b918dba8c42bc67/C of 9806b0fa8f18a18b9b918dba8c42bc67 into 43140d42e3454be4aaee229c3ed44be5(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:58:15,146 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9806b0fa8f18a18b9b918dba8c42bc67: 2024-11-25T12:58:15,146 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., storeName=9806b0fa8f18a18b9b918dba8c42bc67/C, priority=13, startTime=1732539494574; duration=0sec 2024-11-25T12:58:15,147 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:58:15,147 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9806b0fa8f18a18b9b918dba8c42bc67:C 2024-11-25T12:58:15,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:15,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59966 deadline: 1732539555430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:15,441 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:15,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:15,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:15,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:15,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59944 deadline: 1732539555450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:15,453 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 1ff8009e463e,35905,1732539449246 for 'test_row_2' in 9806b0fa8f18a18b9b918dba8c42bc67 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T12:58:15,453 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:15,453 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:15,453 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:15,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:15,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60008 deadline: 1732539555461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:15,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:15,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59968 deadline: 1732539555461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:15,466 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 1ff8009e463e,35905,1732539449246 for 'test_row_2' in 9806b0fa8f18a18b9b918dba8c42bc67 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T12:58:15,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:15,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:15,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:15,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:15,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59962 deadline: 1732539555470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:15,472 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 1ff8009e463e,35905,1732539449246 for 'test_row_0' in 9806b0fa8f18a18b9b918dba8c42bc67 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T12:58:15,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:15,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:15,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:15,472 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 1ff8009e463e,35905,1732539449246 for 'test_row_2' in 9806b0fa8f18a18b9b918dba8c42bc67 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T12:58:15,473 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:15,473 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:15,473 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:15,518 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=210, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/ef98c514cf7d408aad8ca10600c819a7 2024-11-25T12:58:15,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/B/6ad51c8db5614496be55d3b39f0ea3ad is 50, key is test_row_0/B:col10/1732539493547/Put/seqid=0 2024-11-25T12:58:15,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741991_1167 (size=12151) 2024-11-25T12:58:15,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-25T12:58:15,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:15,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59966 deadline: 1732539555952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:15,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:15,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:15,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:15,987 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/B/6ad51c8db5614496be55d3b39f0ea3ad 2024-11-25T12:58:16,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/C/14bb6e453bda4ccaa2725b43d8b15427 is 50, key is test_row_0/C:col10/1732539493547/Put/seqid=0 2024-11-25T12:58:16,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741992_1168 (size=12151) 2024-11-25T12:58:16,458 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/C/14bb6e453bda4ccaa2725b43d8b15427 2024-11-25T12:58:16,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/ef98c514cf7d408aad8ca10600c819a7 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/ef98c514cf7d408aad8ca10600c819a7 2024-11-25T12:58:16,484 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/ef98c514cf7d408aad8ca10600c819a7, entries=150, sequenceid=210, filesize=30.4 K 2024-11-25T12:58:16,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/B/6ad51c8db5614496be55d3b39f0ea3ad as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/6ad51c8db5614496be55d3b39f0ea3ad 2024-11-25T12:58:16,495 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/6ad51c8db5614496be55d3b39f0ea3ad, entries=150, sequenceid=210, filesize=11.9 K 2024-11-25T12:58:16,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/C/14bb6e453bda4ccaa2725b43d8b15427 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/14bb6e453bda4ccaa2725b43d8b15427 2024-11-25T12:58:16,507 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/14bb6e453bda4ccaa2725b43d8b15427, entries=150, sequenceid=210, filesize=11.9 K 2024-11-25T12:58:16,509 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for 9806b0fa8f18a18b9b918dba8c42bc67 in 1858ms, sequenceid=210, compaction requested=false 2024-11-25T12:58:16,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 9806b0fa8f18a18b9b918dba8c42bc67: 2024-11-25T12:58:16,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:16,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-25T12:58:16,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-25T12:58:16,513 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-25T12:58:16,513 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9670 sec 2024-11-25T12:58:16,519 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 2.9750 sec 2024-11-25T12:58:16,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:16,985 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:58:16,985 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9806b0fa8f18a18b9b918dba8c42bc67 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-25T12:58:17,026 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125eb0497a3300245c7b8fa2077c37016d5_9806b0fa8f18a18b9b918dba8c42bc67 is 50, key is test_row_0/A:col10/1732539494773/Put/seqid=0 2024-11-25T12:58:17,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:17,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59966 deadline: 1732539557029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:17,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:17,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:17,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:17,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741993_1169 (size=14794) 2024-11-25T12:58:17,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:17,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59966 deadline: 1732539557140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:17,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:17,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:17,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:17,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:17,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59966 deadline: 1732539557351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:17,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:17,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:17,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:17,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:17,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59944 deadline: 1732539557472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:17,475 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 1ff8009e463e,35905,1732539449246 for 'test_row_2' in 9806b0fa8f18a18b9b918dba8c42bc67 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T12:58:17,476 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:17,476 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:17,476 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:17,486 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:17,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:17,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59968 deadline: 1732539557492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:17,498 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 1ff8009e463e,35905,1732539449246 for 'test_row_2' in 9806b0fa8f18a18b9b918dba8c42bc67 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4238 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T12:58:17,499 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:17,499 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:17,499 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:17,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:17,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60008 deadline: 1732539557492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:17,500 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 1ff8009e463e,35905,1732539449246 for 'test_row_0' in 9806b0fa8f18a18b9b918dba8c42bc67 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4236 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T12:58:17,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:17,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:17,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:17,501 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125eb0497a3300245c7b8fa2077c37016d5_9806b0fa8f18a18b9b918dba8c42bc67 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125eb0497a3300245c7b8fa2077c37016d5_9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:17,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:17,503 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/ef4a7356df454630925a123a5fdcd90c, store: [table=TestAcidGuarantees family=A region=9806b0fa8f18a18b9b918dba8c42bc67] 2024-11-25T12:58:17,503 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/ef4a7356df454630925a123a5fdcd90c is 175, key is test_row_0/A:col10/1732539494773/Put/seqid=0 2024-11-25T12:58:17,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59962 deadline: 1732539557493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:17,505 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 1ff8009e463e,35905,1732539449246 for 'test_row_2' in 9806b0fa8f18a18b9b918dba8c42bc67 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4233 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T12:58:17,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:17,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:17,506 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:17,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741994_1170 (size=39749) 2024-11-25T12:58:17,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:17,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59966 deadline: 1732539557659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:17,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:17,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:17,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:17,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-25T12:58:17,721 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:58:17,722 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:58:17,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-25T12:58:17,723 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:58:17,724 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:58:17,724 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:58:17,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-25T12:58:17,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-25T12:58:17,877 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-25T12:58:17,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:17,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. as already flushing 2024-11-25T12:58:17,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:17,877 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:17,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:17,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:17,951 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=241, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/ef4a7356df454630925a123a5fdcd90c 2024-11-25T12:58:17,969 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/B/d0c4797d673548d7b467573f636f9a78 is 50, key is test_row_0/B:col10/1732539494773/Put/seqid=0 2024-11-25T12:58:18,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741995_1171 (size=12151) 2024-11-25T12:58:18,031 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-25T12:58:18,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:18,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. as already flushing 2024-11-25T12:58:18,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:18,031 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:18,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:18,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:18,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-25T12:58:18,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:18,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59966 deadline: 1732539558171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:18,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:18,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:18,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:18,185 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-25T12:58:18,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:18,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. as already flushing 2024-11-25T12:58:18,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:18,186 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:18,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:18,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:18,341 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-25T12:58:18,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:18,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. as already flushing 2024-11-25T12:58:18,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:18,341 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:18,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:18,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:18,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-25T12:58:18,415 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/B/d0c4797d673548d7b467573f636f9a78 2024-11-25T12:58:18,425 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/C/ac39d140c1ec4411ad31dd826e688ded is 50, key is test_row_0/C:col10/1732539494773/Put/seqid=0 2024-11-25T12:58:18,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741996_1172 (size=12151) 2024-11-25T12:58:18,450 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/C/ac39d140c1ec4411ad31dd826e688ded 2024-11-25T12:58:18,458 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/ef4a7356df454630925a123a5fdcd90c as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/ef4a7356df454630925a123a5fdcd90c 2024-11-25T12:58:18,468 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/ef4a7356df454630925a123a5fdcd90c, entries=200, sequenceid=241, filesize=38.8 K 2024-11-25T12:58:18,472 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/B/d0c4797d673548d7b467573f636f9a78 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/d0c4797d673548d7b467573f636f9a78 2024-11-25T12:58:18,479 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/d0c4797d673548d7b467573f636f9a78, entries=150, sequenceid=241, filesize=11.9 K 2024-11-25T12:58:18,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/C/ac39d140c1ec4411ad31dd826e688ded as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/ac39d140c1ec4411ad31dd826e688ded 2024-11-25T12:58:18,492 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/ac39d140c1ec4411ad31dd826e688ded, entries=150, sequenceid=241, filesize=11.9 K 2024-11-25T12:58:18,493 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 9806b0fa8f18a18b9b918dba8c42bc67 in 1508ms, sequenceid=241, compaction requested=true 2024-11-25T12:58:18,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9806b0fa8f18a18b9b918dba8c42bc67: 2024-11-25T12:58:18,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9806b0fa8f18a18b9b918dba8c42bc67:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:58:18,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:58:18,494 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:58:18,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9806b0fa8f18a18b9b918dba8c42bc67:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:58:18,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:58:18,494 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:58:18,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9806b0fa8f18a18b9b918dba8c42bc67:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:58:18,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:58:18,495 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102403 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:58:18,495 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1541): 9806b0fa8f18a18b9b918dba8c42bc67/A is initiating minor compaction (all files) 2024-11-25T12:58:18,495 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9806b0fa8f18a18b9b918dba8c42bc67/A in TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:18,495 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/620ace6715e54c00abfa1de3f1407cf8, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/ef98c514cf7d408aad8ca10600c819a7, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/ef4a7356df454630925a123a5fdcd90c] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp, totalSize=100.0 K 2024-11-25T12:58:18,496 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:18,496 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. files: [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/620ace6715e54c00abfa1de3f1407cf8, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/ef98c514cf7d408aad8ca10600c819a7, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/ef4a7356df454630925a123a5fdcd90c] 2024-11-25T12:58:18,496 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-25T12:58:18,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:18,496 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 9806b0fa8f18a18b9b918dba8c42bc67 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-25T12:58:18,496 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:58:18,496 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): 9806b0fa8f18a18b9b918dba8c42bc67/B is initiating minor compaction (all files) 2024-11-25T12:58:18,496 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9806b0fa8f18a18b9b918dba8c42bc67/B in TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:18,496 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/daf2b242c2564b7380145d0d5553c1e1, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/6ad51c8db5614496be55d3b39f0ea3ad, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/d0c4797d673548d7b467573f636f9a78] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp, totalSize=36.0 K 2024-11-25T12:58:18,497 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 620ace6715e54c00abfa1de3f1407cf8, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1732539493242 2024-11-25T12:58:18,497 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting daf2b242c2564b7380145d0d5553c1e1, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1732539493242 2024-11-25T12:58:18,497 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef98c514cf7d408aad8ca10600c819a7, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732539493532 2024-11-25T12:58:18,498 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ad51c8db5614496be55d3b39f0ea3ad, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732539493532 2024-11-25T12:58:18,498 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef4a7356df454630925a123a5fdcd90c, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1732539494773 2024-11-25T12:58:18,498 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting d0c4797d673548d7b467573f636f9a78, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1732539494773 2024-11-25T12:58:18,519 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9806b0fa8f18a18b9b918dba8c42bc67#B#compaction#141 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:58:18,520 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/B/ff4d53f662fb4bebb69da3f19abf750d is 50, key is test_row_0/B:col10/1732539494773/Put/seqid=0 2024-11-25T12:58:18,528 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9806b0fa8f18a18b9b918dba8c42bc67] 2024-11-25T12:58:18,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125afd305103edf41f785c1110cfd7fda3f_9806b0fa8f18a18b9b918dba8c42bc67 is 50, key is test_row_0/A:col10/1732539496992/Put/seqid=0 2024-11-25T12:58:18,581 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241125fbd92218e9ec4601915568c9c14b73f2_9806b0fa8f18a18b9b918dba8c42bc67 store=[table=TestAcidGuarantees family=A region=9806b0fa8f18a18b9b918dba8c42bc67] 2024-11-25T12:58:18,583 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241125fbd92218e9ec4601915568c9c14b73f2_9806b0fa8f18a18b9b918dba8c42bc67, store=[table=TestAcidGuarantees family=A region=9806b0fa8f18a18b9b918dba8c42bc67] 2024-11-25T12:58:18,583 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125fbd92218e9ec4601915568c9c14b73f2_9806b0fa8f18a18b9b918dba8c42bc67 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9806b0fa8f18a18b9b918dba8c42bc67] 2024-11-25T12:58:18,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741998_1174 (size=12304) 2024-11-25T12:58:18,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741997_1173 (size=12697) 2024-11-25T12:58:18,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741999_1175 (size=4469) 2024-11-25T12:58:18,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-25T12:58:19,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:19,027 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125afd305103edf41f785c1110cfd7fda3f_9806b0fa8f18a18b9b918dba8c42bc67 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125afd305103edf41f785c1110cfd7fda3f_9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:19,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/27c4f6483e414fc198832c96dbf8d042, store: [table=TestAcidGuarantees family=A region=9806b0fa8f18a18b9b918dba8c42bc67] 2024-11-25T12:58:19,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/27c4f6483e414fc198832c96dbf8d042 is 175, key is test_row_0/A:col10/1732539496992/Put/seqid=0 2024-11-25T12:58:19,030 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/B/ff4d53f662fb4bebb69da3f19abf750d as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/ff4d53f662fb4bebb69da3f19abf750d 2024-11-25T12:58:19,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742000_1176 (size=31105) 2024-11-25T12:58:19,035 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9806b0fa8f18a18b9b918dba8c42bc67/B of 9806b0fa8f18a18b9b918dba8c42bc67 into ff4d53f662fb4bebb69da3f19abf750d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:58:19,036 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9806b0fa8f18a18b9b918dba8c42bc67: 2024-11-25T12:58:19,036 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., storeName=9806b0fa8f18a18b9b918dba8c42bc67/B, priority=13, startTime=1732539498494; duration=0sec 2024-11-25T12:58:19,036 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:58:19,036 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9806b0fa8f18a18b9b918dba8c42bc67:B 2024-11-25T12:58:19,036 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:58:19,037 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:58:19,038 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): 9806b0fa8f18a18b9b918dba8c42bc67/C is initiating minor compaction (all files) 2024-11-25T12:58:19,038 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=249, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/27c4f6483e414fc198832c96dbf8d042 2024-11-25T12:58:19,038 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9806b0fa8f18a18b9b918dba8c42bc67/C in TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:19,038 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/43140d42e3454be4aaee229c3ed44be5, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/14bb6e453bda4ccaa2725b43d8b15427, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/ac39d140c1ec4411ad31dd826e688ded] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp, totalSize=36.0 K 2024-11-25T12:58:19,038 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 43140d42e3454be4aaee229c3ed44be5, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1732539493242 2024-11-25T12:58:19,039 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 14bb6e453bda4ccaa2725b43d8b15427, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732539493532 2024-11-25T12:58:19,040 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting ac39d140c1ec4411ad31dd826e688ded, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1732539494773 2024-11-25T12:58:19,045 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9806b0fa8f18a18b9b918dba8c42bc67#A#compaction#142 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:58:19,045 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/a99855dc5ceb4a5680df06abce0434b0 is 175, key is test_row_0/A:col10/1732539494773/Put/seqid=0 2024-11-25T12:58:19,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/B/2c6851f994654fe08d69e38b79755a00 is 50, key is test_row_0/B:col10/1732539496992/Put/seqid=0 2024-11-25T12:58:19,064 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9806b0fa8f18a18b9b918dba8c42bc67#C#compaction#145 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:58:19,065 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/C/8663bc1835b14073a2ee92d3f99e1da6 is 50, key is test_row_0/C:col10/1732539494773/Put/seqid=0 2024-11-25T12:58:19,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742001_1177 (size=31651) 2024-11-25T12:58:19,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742003_1179 (size=12697) 2024-11-25T12:58:19,125 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/C/8663bc1835b14073a2ee92d3f99e1da6 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/8663bc1835b14073a2ee92d3f99e1da6 2024-11-25T12:58:19,130 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9806b0fa8f18a18b9b918dba8c42bc67/C of 9806b0fa8f18a18b9b918dba8c42bc67 into 8663bc1835b14073a2ee92d3f99e1da6(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:58:19,130 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9806b0fa8f18a18b9b918dba8c42bc67: 2024-11-25T12:58:19,130 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., storeName=9806b0fa8f18a18b9b918dba8c42bc67/C, priority=13, startTime=1732539498494; duration=0sec 2024-11-25T12:58:19,130 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:58:19,130 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9806b0fa8f18a18b9b918dba8c42bc67:C 2024-11-25T12:58:19,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742002_1178 (size=12151) 2024-11-25T12:58:19,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. as already flushing 2024-11-25T12:58:19,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:19,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:19,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59966 deadline: 1732539559291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:19,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:19,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:19,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:19,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:19,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59966 deadline: 1732539559410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:19,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:19,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:19,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:19,525 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/a99855dc5ceb4a5680df06abce0434b0 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/a99855dc5ceb4a5680df06abce0434b0 2024-11-25T12:58:19,539 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/B/2c6851f994654fe08d69e38b79755a00 2024-11-25T12:58:19,542 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9806b0fa8f18a18b9b918dba8c42bc67/A of 9806b0fa8f18a18b9b918dba8c42bc67 into a99855dc5ceb4a5680df06abce0434b0(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 1sec to execute. 2024-11-25T12:58:19,542 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9806b0fa8f18a18b9b918dba8c42bc67: 2024-11-25T12:58:19,542 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., storeName=9806b0fa8f18a18b9b918dba8c42bc67/A, priority=13, startTime=1732539498493; duration=1sec 2024-11-25T12:58:19,542 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:58:19,543 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9806b0fa8f18a18b9b918dba8c42bc67:A 2024-11-25T12:58:19,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/C/d459d7f867db4fcb8b60b8b52c210053 is 50, key is test_row_0/C:col10/1732539496992/Put/seqid=0 2024-11-25T12:58:19,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742004_1180 (size=12151) 2024-11-25T12:58:19,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:19,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59966 deadline: 1732539559620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:19,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:19,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:19,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:19,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-25T12:58:19,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:19,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59966 deadline: 1732539559931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:19,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:19,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:19,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:19,967 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/C/d459d7f867db4fcb8b60b8b52c210053 2024-11-25T12:58:19,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/27c4f6483e414fc198832c96dbf8d042 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/27c4f6483e414fc198832c96dbf8d042 2024-11-25T12:58:19,997 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/27c4f6483e414fc198832c96dbf8d042, entries=150, sequenceid=249, filesize=30.4 K 2024-11-25T12:58:20,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/B/2c6851f994654fe08d69e38b79755a00 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/2c6851f994654fe08d69e38b79755a00 2024-11-25T12:58:20,010 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/2c6851f994654fe08d69e38b79755a00, entries=150, sequenceid=249, filesize=11.9 K 2024-11-25T12:58:20,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/C/d459d7f867db4fcb8b60b8b52c210053 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/d459d7f867db4fcb8b60b8b52c210053 2024-11-25T12:58:20,032 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/d459d7f867db4fcb8b60b8b52c210053, entries=150, sequenceid=249, filesize=11.9 K 2024-11-25T12:58:20,035 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=167.72 KB/171750 for 9806b0fa8f18a18b9b918dba8c42bc67 in 1539ms, sequenceid=249, compaction requested=false 2024-11-25T12:58:20,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 9806b0fa8f18a18b9b918dba8c42bc67: 2024-11-25T12:58:20,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:20,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-25T12:58:20,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-25T12:58:20,039 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-25T12:58:20,039 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3130 sec 2024-11-25T12:58:20,041 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 2.3180 sec 2024-11-25T12:58:20,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:20,452 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:58:20,452 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9806b0fa8f18a18b9b918dba8c42bc67 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-25T12:58:20,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:20,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59966 deadline: 1732539560465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:20,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:20,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:20,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:20,473 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125c5bbdcbe853d4f799a021104684ce2f4_9806b0fa8f18a18b9b918dba8c42bc67 is 50, key is test_row_0/A:col10/1732539499289/Put/seqid=0 2024-11-25T12:58:20,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742005_1181 (size=14994) 2024-11-25T12:58:20,489 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:20,510 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125c5bbdcbe853d4f799a021104684ce2f4_9806b0fa8f18a18b9b918dba8c42bc67 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125c5bbdcbe853d4f799a021104684ce2f4_9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:20,511 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/a13208199b614024a57666c84e2eeff8, store: [table=TestAcidGuarantees family=A region=9806b0fa8f18a18b9b918dba8c42bc67] 2024-11-25T12:58:20,512 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/a13208199b614024a57666c84e2eeff8 is 175, key is test_row_0/A:col10/1732539499289/Put/seqid=0 2024-11-25T12:58:20,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742006_1182 (size=39949) 2024-11-25T12:58:20,518 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=281, memsize=58.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/a13208199b614024a57666c84e2eeff8 2024-11-25T12:58:20,529 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/B/4c5306706dbb4a9e8c9c104145f2bf2d is 50, key is test_row_0/B:col10/1732539499289/Put/seqid=0 2024-11-25T12:58:20,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742007_1183 (size=12301) 2024-11-25T12:58:20,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:20,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59966 deadline: 1732539560570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:20,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:20,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:20,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:20,573 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/B/4c5306706dbb4a9e8c9c104145f2bf2d 2024-11-25T12:58:20,597 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/C/b62e0ea79cb4430889f80c36d187ae59 is 50, key is test_row_0/C:col10/1732539499289/Put/seqid=0 2024-11-25T12:58:20,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742008_1184 (size=12301) 2024-11-25T12:58:20,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:20,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59966 deadline: 1732539560780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:20,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:20,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:20,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:21,024 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/C/b62e0ea79cb4430889f80c36d187ae59 2024-11-25T12:58:21,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/a13208199b614024a57666c84e2eeff8 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/a13208199b614024a57666c84e2eeff8 2024-11-25T12:58:21,052 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/a13208199b614024a57666c84e2eeff8, entries=200, sequenceid=281, filesize=39.0 K 2024-11-25T12:58:21,054 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/B/4c5306706dbb4a9e8c9c104145f2bf2d as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/4c5306706dbb4a9e8c9c104145f2bf2d 2024-11-25T12:58:21,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,062 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/4c5306706dbb4a9e8c9c104145f2bf2d, entries=150, sequenceid=281, filesize=12.0 K 2024-11-25T12:58:21,064 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/C/b62e0ea79cb4430889f80c36d187ae59 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/b62e0ea79cb4430889f80c36d187ae59 2024-11-25T12:58:21,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,072 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/b62e0ea79cb4430889f80c36d187ae59, entries=150, sequenceid=281, filesize=12.0 K 2024-11-25T12:58:21,074 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=26.84 KB/27480 for 9806b0fa8f18a18b9b918dba8c42bc67 in 621ms, sequenceid=281, compaction requested=true 2024-11-25T12:58:21,074 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9806b0fa8f18a18b9b918dba8c42bc67: 2024-11-25T12:58:21,074 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:58:21,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9806b0fa8f18a18b9b918dba8c42bc67:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:58:21,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:58:21,076 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:58:21,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9806b0fa8f18a18b9b918dba8c42bc67:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:58:21,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:58:21,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9806b0fa8f18a18b9b918dba8c42bc67:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:58:21,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:58:21,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,080 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102705 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:58:21,080 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1541): 9806b0fa8f18a18b9b918dba8c42bc67/A is initiating minor compaction (all files) 2024-11-25T12:58:21,080 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9806b0fa8f18a18b9b918dba8c42bc67/A in TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:21,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,080 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/a99855dc5ceb4a5680df06abce0434b0, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/27c4f6483e414fc198832c96dbf8d042, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/a13208199b614024a57666c84e2eeff8] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp, totalSize=100.3 K 2024-11-25T12:58:21,080 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:21,080 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. files: [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/a99855dc5ceb4a5680df06abce0434b0, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/27c4f6483e414fc198832c96dbf8d042, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/a13208199b614024a57666c84e2eeff8] 2024-11-25T12:58:21,081 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting a99855dc5ceb4a5680df06abce0434b0, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1732539494773 2024-11-25T12:58:21,081 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27c4f6483e414fc198832c96dbf8d042, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1732539496992 2024-11-25T12:58:21,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,081 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:58:21,081 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): 9806b0fa8f18a18b9b918dba8c42bc67/B is initiating minor compaction (all files) 2024-11-25T12:58:21,081 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9806b0fa8f18a18b9b918dba8c42bc67/B in TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:21,081 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/ff4d53f662fb4bebb69da3f19abf750d, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/2c6851f994654fe08d69e38b79755a00, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/4c5306706dbb4a9e8c9c104145f2bf2d] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp, totalSize=36.3 K 2024-11-25T12:58:21,082 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting a13208199b614024a57666c84e2eeff8, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1732539499261 2024-11-25T12:58:21,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,082 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting ff4d53f662fb4bebb69da3f19abf750d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1732539494773 2024-11-25T12:58:21,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,083 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c6851f994654fe08d69e38b79755a00, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1732539496992 2024-11-25T12:58:21,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,088 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c5306706dbb4a9e8c9c104145f2bf2d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1732539499261 2024-11-25T12:58:21,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,106 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9806b0fa8f18a18b9b918dba8c42bc67] 2024-11-25T12:58:21,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,124 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112561cc61d1d43f4294bed935a4b6459301_9806b0fa8f18a18b9b918dba8c42bc67 store=[table=TestAcidGuarantees family=A region=9806b0fa8f18a18b9b918dba8c42bc67] 2024-11-25T12:58:21,127 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9806b0fa8f18a18b9b918dba8c42bc67#B#compaction#151 average throughput is 0.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:58:21,127 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/B/6b4c129c0e2644869731b7a5f8885904 is 50, key is test_row_0/B:col10/1732539499289/Put/seqid=0 2024-11-25T12:58:21,132 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112561cc61d1d43f4294bed935a4b6459301_9806b0fa8f18a18b9b918dba8c42bc67, store=[table=TestAcidGuarantees family=A region=9806b0fa8f18a18b9b918dba8c42bc67] 2024-11-25T12:58:21,132 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112561cc61d1d43f4294bed935a4b6459301_9806b0fa8f18a18b9b918dba8c42bc67 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9806b0fa8f18a18b9b918dba8c42bc67] 2024-11-25T12:58:21,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742009_1185 (size=12949) 2024-11-25T12:58:21,215 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:58:21,215 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9806b0fa8f18a18b9b918dba8c42bc67 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-25T12:58:21,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:21,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742010_1186 (size=4469) 2024-11-25T12:58:21,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,226 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9806b0fa8f18a18b9b918dba8c42bc67#A#compaction#150 average throughput is 0.20 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:58:21,226 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/cb8a727d82904cb9b6989ab1ad670885 is 175, key is test_row_0/A:col10/1732539499289/Put/seqid=0 2024-11-25T12:58:21,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,256 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411258bcc9d8f8f704f318a0335dc3b79a100_9806b0fa8f18a18b9b918dba8c42bc67 is 50, key is test_row_0/A:col10/1732539501139/Put/seqid=0 2024-11-25T12:58:21,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742011_1187 (size=31903) 2024-11-25T12:58:21,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,290 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/cb8a727d82904cb9b6989ab1ad670885 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/cb8a727d82904cb9b6989ab1ad670885 2024-11-25T12:58:21,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,298 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9806b0fa8f18a18b9b918dba8c42bc67/A of 9806b0fa8f18a18b9b918dba8c42bc67 into cb8a727d82904cb9b6989ab1ad670885(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:58:21,298 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9806b0fa8f18a18b9b918dba8c42bc67: 2024-11-25T12:58:21,298 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., storeName=9806b0fa8f18a18b9b918dba8c42bc67/A, priority=13, startTime=1732539501074; duration=0sec 2024-11-25T12:58:21,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,300 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:58:21,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,302 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9806b0fa8f18a18b9b918dba8c42bc67:A 2024-11-25T12:58:21,303 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:58:21,304 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:58:21,304 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1541): 9806b0fa8f18a18b9b918dba8c42bc67/C is initiating minor compaction (all files) 2024-11-25T12:58:21,304 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9806b0fa8f18a18b9b918dba8c42bc67/C in TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:21,305 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/8663bc1835b14073a2ee92d3f99e1da6, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/d459d7f867db4fcb8b60b8b52c210053, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/b62e0ea79cb4430889f80c36d187ae59] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp, totalSize=36.3 K 2024-11-25T12:58:21,305 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8663bc1835b14073a2ee92d3f99e1da6, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1732539494773 2024-11-25T12:58:21,306 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting d459d7f867db4fcb8b60b8b52c210053, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1732539496992 2024-11-25T12:58:21,306 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting b62e0ea79cb4430889f80c36d187ae59, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1732539499261 2024-11-25T12:58:21,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742012_1188 (size=20074) 2024-11-25T12:58:21,329 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,337 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411258bcc9d8f8f704f318a0335dc3b79a100_9806b0fa8f18a18b9b918dba8c42bc67 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411258bcc9d8f8f704f318a0335dc3b79a100_9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:21,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,340 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/3600d5198fa84231aa24778d39f97a68, store: [table=TestAcidGuarantees family=A region=9806b0fa8f18a18b9b918dba8c42bc67] 2024-11-25T12:58:21,341 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/3600d5198fa84231aa24778d39f97a68 is 175, key is test_row_0/A:col10/1732539501139/Put/seqid=0 2024-11-25T12:58:21,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,366 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9806b0fa8f18a18b9b918dba8c42bc67#C#compaction#153 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:58:21,366 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/C/ffb36df48ed6414ca6fbd91a664a65e8 is 50, key is test_row_0/C:col10/1732539499289/Put/seqid=0 2024-11-25T12:58:21,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742013_1189 (size=57329) 2024-11-25T12:58:21,401 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=292, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/3600d5198fa84231aa24778d39f97a68 2024-11-25T12:58:21,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742014_1190 (size=12949) 2024-11-25T12:58:21,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,447 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/B/720795ad923e44acbf5828a6440b9f67 is 50, key is test_row_0/B:col10/1732539501139/Put/seqid=0 2024-11-25T12:58:21,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,451 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/C/ffb36df48ed6414ca6fbd91a664a65e8 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/ffb36df48ed6414ca6fbd91a664a65e8 2024-11-25T12:58:21,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,458 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9806b0fa8f18a18b9b918dba8c42bc67/C of 9806b0fa8f18a18b9b918dba8c42bc67 into ffb36df48ed6414ca6fbd91a664a65e8(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:58:21,459 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9806b0fa8f18a18b9b918dba8c42bc67: 2024-11-25T12:58:21,459 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., storeName=9806b0fa8f18a18b9b918dba8c42bc67/C, priority=13, startTime=1732539501076; duration=0sec 2024-11-25T12:58:21,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,459 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:58:21,459 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9806b0fa8f18a18b9b918dba8c42bc67:C 2024-11-25T12:58:21,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:21,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742015_1191 (size=9857) 2024-11-25T12:58:21,477 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/B/720795ad923e44acbf5828a6440b9f67 2024-11-25T12:58:21,487 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/C/dc25906560dd4aae9cd770ae83f35a89 is 50, key is test_row_0/C:col10/1732539501139/Put/seqid=0 2024-11-25T12:58:21,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742016_1192 (size=9857) 2024-11-25T12:58:21,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:21,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59966 deadline: 1732539561523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:21,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:21,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:21,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:21,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59968 deadline: 1732539561525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:21,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:21,526 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:21,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:21,526 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:21,526 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:21,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59944 deadline: 1732539561525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:21,527 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:21,527 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:21,527 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:21,534 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:21,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59962 deadline: 1732539561530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:21,534 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:21,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60008 deadline: 1732539561530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:21,534 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 1ff8009e463e,35905,1732539449246 for 'test_row_2' in 9806b0fa8f18a18b9b918dba8c42bc67 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8263 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T12:58:21,535 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:21,535 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:21,535 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:21,535 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 1ff8009e463e,35905,1732539449246 for 'test_row_0' in 9806b0fa8f18a18b9b918dba8c42bc67 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8271 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T12:58:21,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:21,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:21,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:21,616 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/B/6b4c129c0e2644869731b7a5f8885904 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/6b4c129c0e2644869731b7a5f8885904 2024-11-25T12:58:21,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:21,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59966 deadline: 1732539561630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:21,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:21,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:21,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:21,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:21,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:21,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59968 deadline: 1732539561631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:21,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59944 deadline: 1732539561630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:21,633 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:21,633 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:21,633 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:21,633 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9806b0fa8f18a18b9b918dba8c42bc67/B of 9806b0fa8f18a18b9b918dba8c42bc67 into 6b4c129c0e2644869731b7a5f8885904(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:58:21,633 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9806b0fa8f18a18b9b918dba8c42bc67: 2024-11-25T12:58:21,633 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:21,634 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:21,634 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:21,634 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., storeName=9806b0fa8f18a18b9b918dba8c42bc67/B, priority=13, startTime=1732539501076; duration=0sec 2024-11-25T12:58:21,634 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:58:21,634 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9806b0fa8f18a18b9b918dba8c42bc67:B 2024-11-25T12:58:21,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:21,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:21,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59966 deadline: 1732539561840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:21,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59968 deadline: 1732539561840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:21,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:21,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59944 deadline: 1732539561840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:21,845 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:21,845 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:21,845 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:21,845 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:21,845 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:21,845 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:21,845 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:21,845 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:21,845 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:21,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-25T12:58:21,890 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:58:21,891 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:58:21,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-25T12:58:21,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-25T12:58:21,901 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/C/dc25906560dd4aae9cd770ae83f35a89 2024-11-25T12:58:21,904 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:58:21,905 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:58:21,905 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:58:21,911 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/3600d5198fa84231aa24778d39f97a68 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/3600d5198fa84231aa24778d39f97a68 2024-11-25T12:58:21,918 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/3600d5198fa84231aa24778d39f97a68, entries=300, sequenceid=292, filesize=56.0 K 2024-11-25T12:58:21,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/B/720795ad923e44acbf5828a6440b9f67 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/720795ad923e44acbf5828a6440b9f67 2024-11-25T12:58:21,953 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/720795ad923e44acbf5828a6440b9f67, entries=100, sequenceid=292, filesize=9.6 K 2024-11-25T12:58:21,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/C/dc25906560dd4aae9cd770ae83f35a89 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/dc25906560dd4aae9cd770ae83f35a89 2024-11-25T12:58:21,975 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/dc25906560dd4aae9cd770ae83f35a89, entries=100, sequenceid=292, filesize=9.6 K 2024-11-25T12:58:21,976 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 9806b0fa8f18a18b9b918dba8c42bc67 in 761ms, sequenceid=292, compaction requested=false 2024-11-25T12:58:21,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9806b0fa8f18a18b9b918dba8c42bc67: 2024-11-25T12:58:22,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-25T12:58:22,059 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-25T12:58:22,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:22,060 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 9806b0fa8f18a18b9b918dba8c42bc67 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-25T12:58:22,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125ee4be1eec10a4442a63fdce76cce6624_9806b0fa8f18a18b9b918dba8c42bc67 is 50, key is test_row_0/A:col10/1732539501521/Put/seqid=0 2024-11-25T12:58:22,162 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. as already flushing 2024-11-25T12:58:22,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:22,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742017_1193 (size=12454) 2024-11-25T12:58:22,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:22,169 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125ee4be1eec10a4442a63fdce76cce6624_9806b0fa8f18a18b9b918dba8c42bc67 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125ee4be1eec10a4442a63fdce76cce6624_9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:22,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/2692d92740e440caa6a5bf96faec6c70, store: [table=TestAcidGuarantees family=A region=9806b0fa8f18a18b9b918dba8c42bc67] 2024-11-25T12:58:22,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/2692d92740e440caa6a5bf96faec6c70 is 175, key is test_row_0/A:col10/1732539501521/Put/seqid=0 2024-11-25T12:58:22,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:22,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59944 deadline: 1732539562178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:22,186 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:22,186 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:22,186 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:22,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:22,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59966 deadline: 1732539562181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:22,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:22,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:22,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:22,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:22,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59968 deadline: 1732539562183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:22,191 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:22,191 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:22,191 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:22,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742018_1194 (size=31255) 2024-11-25T12:58:22,204 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=320, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/2692d92740e440caa6a5bf96faec6c70 2024-11-25T12:58:22,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-25T12:58:22,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/B/ae30b18ed26242edb7c44fe8fcf1f035 is 50, key is test_row_0/B:col10/1732539501521/Put/seqid=0 2024-11-25T12:58:22,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742019_1195 (size=12301) 2024-11-25T12:58:22,258 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/B/ae30b18ed26242edb7c44fe8fcf1f035 2024-11-25T12:58:22,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/C/fe2206c0c3694f9e99d891fa23d00bbd is 50, key is test_row_0/C:col10/1732539501521/Put/seqid=0 2024-11-25T12:58:22,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:22,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59944 deadline: 1732539562295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:22,301 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:22,301 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:22,301 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:22,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:22,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59966 deadline: 1732539562300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:22,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:22,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:22,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:22,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:22,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59968 deadline: 1732539562307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:22,313 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:22,314 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:22,314 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:22,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742020_1196 (size=12301) 2024-11-25T12:58:22,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:22,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59944 deadline: 1732539562510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:22,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:22,517 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:22,517 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:22,517 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:22,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59966 deadline: 1732539562510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:22,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:22,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:22,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:22,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-25T12:58:22,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:22,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59968 deadline: 1732539562522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:22,531 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:22,531 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:22,531 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:22,766 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/C/fe2206c0c3694f9e99d891fa23d00bbd 2024-11-25T12:58:22,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/2692d92740e440caa6a5bf96faec6c70 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/2692d92740e440caa6a5bf96faec6c70 2024-11-25T12:58:22,797 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/2692d92740e440caa6a5bf96faec6c70, entries=150, sequenceid=320, filesize=30.5 K 2024-11-25T12:58:22,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/B/ae30b18ed26242edb7c44fe8fcf1f035 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/ae30b18ed26242edb7c44fe8fcf1f035 2024-11-25T12:58:22,805 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/ae30b18ed26242edb7c44fe8fcf1f035, entries=150, sequenceid=320, filesize=12.0 K 2024-11-25T12:58:22,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/C/fe2206c0c3694f9e99d891fa23d00bbd as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/fe2206c0c3694f9e99d891fa23d00bbd 2024-11-25T12:58:22,813 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/fe2206c0c3694f9e99d891fa23d00bbd, entries=150, sequenceid=320, filesize=12.0 K 2024-11-25T12:58:22,814 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 9806b0fa8f18a18b9b918dba8c42bc67 in 754ms, sequenceid=320, compaction requested=true 2024-11-25T12:58:22,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 9806b0fa8f18a18b9b918dba8c42bc67: 2024-11-25T12:58:22,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:22,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-25T12:58:22,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-25T12:58:22,818 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-25T12:58:22,818 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 912 msec 2024-11-25T12:58:22,820 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 928 msec 2024-11-25T12:58:22,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:22,824 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:58:22,825 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9806b0fa8f18a18b9b918dba8c42bc67 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-25T12:58:22,845 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411258fcf2b80c8134fc2873c083a1e8cdc3c_9806b0fa8f18a18b9b918dba8c42bc67 is 50, key is test_row_0/A:col10/1732539502181/Put/seqid=0 2024-11-25T12:58:22,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742021_1197 (size=12454) 2024-11-25T12:58:22,880 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:22,886 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411258fcf2b80c8134fc2873c083a1e8cdc3c_9806b0fa8f18a18b9b918dba8c42bc67 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411258fcf2b80c8134fc2873c083a1e8cdc3c_9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:22,887 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/4e218e975942401cb4fac3c4558efa20, store: [table=TestAcidGuarantees family=A region=9806b0fa8f18a18b9b918dba8c42bc67] 2024-11-25T12:58:22,887 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/4e218e975942401cb4fac3c4558efa20 is 175, key is test_row_0/A:col10/1732539502181/Put/seqid=0 2024-11-25T12:58:22,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:22,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59944 deadline: 1732539562884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:22,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:22,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59968 deadline: 1732539562885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:22,892 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:22,892 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:22,892 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:22,893 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:22,893 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:22,893 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:22,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:22,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59966 deadline: 1732539562892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:22,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:22,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:22,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:22,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742022_1198 (size=31255) 2024-11-25T12:58:22,909 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=332, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/4e218e975942401cb4fac3c4558efa20 2024-11-25T12:58:22,919 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/B/77c4c2b4d9414cf5aaab19403845fff7 is 50, key is test_row_0/B:col10/1732539502181/Put/seqid=0 2024-11-25T12:58:22,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742023_1199 (size=12301) 2024-11-25T12:58:22,940 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/B/77c4c2b4d9414cf5aaab19403845fff7 2024-11-25T12:58:22,957 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/C/e597d8e21ab0450e817c9bfd46cffc30 is 50, key is test_row_0/C:col10/1732539502181/Put/seqid=0 2024-11-25T12:58:22,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742024_1200 (size=12301) 2024-11-25T12:58:23,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:23,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59968 deadline: 1732539562999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:23,001 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:23,002 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:23,002 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:23,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:23,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59944 deadline: 1732539563000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:23,002 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:23,002 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:23,002 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:23,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:23,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59966 deadline: 1732539563000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:23,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:23,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:23,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:23,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-25T12:58:23,030 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:58:23,031 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:58:23,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-25T12:58:23,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-25T12:58:23,033 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:58:23,034 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:58:23,034 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:58:23,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-25T12:58:23,188 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-25T12:58:23,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:23,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. as already flushing 2024-11-25T12:58:23,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:23,188 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:23,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:23,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:23,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:23,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59968 deadline: 1732539563210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:23,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:23,217 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:23,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59944 deadline: 1732539563210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:23,217 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:23,217 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:23,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:23,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59966 deadline: 1732539563210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:23,217 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:23,217 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:23,217 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:23,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:23,218 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:23,218 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:23,344 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-25T12:58:23,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:23,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. as already flushing 2024-11-25T12:58:23,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:23,345 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:23,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:23,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:23,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-25T12:58:23,391 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/C/e597d8e21ab0450e817c9bfd46cffc30 2024-11-25T12:58:23,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/4e218e975942401cb4fac3c4558efa20 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/4e218e975942401cb4fac3c4558efa20 2024-11-25T12:58:23,433 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/4e218e975942401cb4fac3c4558efa20, entries=150, sequenceid=332, filesize=30.5 K 2024-11-25T12:58:23,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/B/77c4c2b4d9414cf5aaab19403845fff7 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/77c4c2b4d9414cf5aaab19403845fff7 2024-11-25T12:58:23,448 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/77c4c2b4d9414cf5aaab19403845fff7, entries=150, sequenceid=332, filesize=12.0 K 2024-11-25T12:58:23,450 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/C/e597d8e21ab0450e817c9bfd46cffc30 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/e597d8e21ab0450e817c9bfd46cffc30 2024-11-25T12:58:23,455 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/e597d8e21ab0450e817c9bfd46cffc30, entries=150, sequenceid=332, filesize=12.0 K 2024-11-25T12:58:23,457 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 9806b0fa8f18a18b9b918dba8c42bc67 in 633ms, sequenceid=332, compaction requested=true 2024-11-25T12:58:23,457 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9806b0fa8f18a18b9b918dba8c42bc67: 2024-11-25T12:58:23,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9806b0fa8f18a18b9b918dba8c42bc67:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:58:23,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:58:23,458 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:58:23,458 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:58:23,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9806b0fa8f18a18b9b918dba8c42bc67:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:58:23,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:58:23,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9806b0fa8f18a18b9b918dba8c42bc67:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:58:23,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:58:23,460 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 151742 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:58:23,460 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1541): 9806b0fa8f18a18b9b918dba8c42bc67/A is initiating minor compaction (all files) 2024-11-25T12:58:23,460 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9806b0fa8f18a18b9b918dba8c42bc67/A in TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:23,460 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/cb8a727d82904cb9b6989ab1ad670885, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/3600d5198fa84231aa24778d39f97a68, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/2692d92740e440caa6a5bf96faec6c70, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/4e218e975942401cb4fac3c4558efa20] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp, totalSize=148.2 K 2024-11-25T12:58:23,460 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:23,460 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. files: [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/cb8a727d82904cb9b6989ab1ad670885, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/3600d5198fa84231aa24778d39f97a68, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/2692d92740e440caa6a5bf96faec6c70, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/4e218e975942401cb4fac3c4558efa20] 2024-11-25T12:58:23,461 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47408 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:58:23,461 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb8a727d82904cb9b6989ab1ad670885, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1732539499261 2024-11-25T12:58:23,461 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): 9806b0fa8f18a18b9b918dba8c42bc67/B is initiating minor compaction (all files) 2024-11-25T12:58:23,461 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9806b0fa8f18a18b9b918dba8c42bc67/B in TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:23,461 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/6b4c129c0e2644869731b7a5f8885904, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/720795ad923e44acbf5828a6440b9f67, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/ae30b18ed26242edb7c44fe8fcf1f035, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/77c4c2b4d9414cf5aaab19403845fff7] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp, totalSize=46.3 K 2024-11-25T12:58:23,461 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3600d5198fa84231aa24778d39f97a68, keycount=300, bloomtype=ROW, size=56.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1732539500455 2024-11-25T12:58:23,462 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b4c129c0e2644869731b7a5f8885904, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1732539499261 2024-11-25T12:58:23,462 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2692d92740e440caa6a5bf96faec6c70, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732539501521 2024-11-25T12:58:23,462 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 720795ad923e44acbf5828a6440b9f67, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1732539501139 2024-11-25T12:58:23,462 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e218e975942401cb4fac3c4558efa20, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1732539502179 2024-11-25T12:58:23,462 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting ae30b18ed26242edb7c44fe8fcf1f035, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732539501521 2024-11-25T12:58:23,463 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 77c4c2b4d9414cf5aaab19403845fff7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1732539502179 2024-11-25T12:58:23,476 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9806b0fa8f18a18b9b918dba8c42bc67] 2024-11-25T12:58:23,490 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9806b0fa8f18a18b9b918dba8c42bc67#B#compaction#163 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:58:23,490 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/B/4f907b83f8be4beea6cc888f6b12a0d9 is 50, key is test_row_0/B:col10/1732539502181/Put/seqid=0 2024-11-25T12:58:23,502 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-25T12:58:23,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:23,503 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 9806b0fa8f18a18b9b918dba8c42bc67 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-25T12:58:23,516 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411255c13669a2679466d9fd24c5b96261c12_9806b0fa8f18a18b9b918dba8c42bc67 store=[table=TestAcidGuarantees family=A region=9806b0fa8f18a18b9b918dba8c42bc67] 2024-11-25T12:58:23,518 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411255c13669a2679466d9fd24c5b96261c12_9806b0fa8f18a18b9b918dba8c42bc67, store=[table=TestAcidGuarantees family=A region=9806b0fa8f18a18b9b918dba8c42bc67] 2024-11-25T12:58:23,519 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411255c13669a2679466d9fd24c5b96261c12_9806b0fa8f18a18b9b918dba8c42bc67 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9806b0fa8f18a18b9b918dba8c42bc67] 2024-11-25T12:58:23,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:23,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. as already flushing 2024-11-25T12:58:23,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:23,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59966 deadline: 1732539563540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:23,550 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:23,550 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:23,550 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:23,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:23,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59968 deadline: 1732539563546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:23,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:23,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59944 deadline: 1732539563546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:23,552 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:23,552 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:23,552 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:23,552 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:23,552 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:23,552 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:23,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742025_1201 (size=13085) 2024-11-25T12:58:23,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411253c3926abc0cb441a899219fc8d15ea62_9806b0fa8f18a18b9b918dba8c42bc67 is 50, key is test_row_0/A:col10/1732539502879/Put/seqid=0 2024-11-25T12:58:23,582 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/B/4f907b83f8be4beea6cc888f6b12a0d9 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/4f907b83f8be4beea6cc888f6b12a0d9 2024-11-25T12:58:23,598 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9806b0fa8f18a18b9b918dba8c42bc67/B of 9806b0fa8f18a18b9b918dba8c42bc67 into 4f907b83f8be4beea6cc888f6b12a0d9(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:58:23,598 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9806b0fa8f18a18b9b918dba8c42bc67: 2024-11-25T12:58:23,598 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., storeName=9806b0fa8f18a18b9b918dba8c42bc67/B, priority=12, startTime=1732539503458; duration=0sec 2024-11-25T12:58:23,599 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:58:23,599 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9806b0fa8f18a18b9b918dba8c42bc67:B 2024-11-25T12:58:23,599 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:58:23,601 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47408 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:58:23,601 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): 9806b0fa8f18a18b9b918dba8c42bc67/C is initiating minor compaction (all files) 2024-11-25T12:58:23,601 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9806b0fa8f18a18b9b918dba8c42bc67/C in TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:23,602 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/ffb36df48ed6414ca6fbd91a664a65e8, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/dc25906560dd4aae9cd770ae83f35a89, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/fe2206c0c3694f9e99d891fa23d00bbd, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/e597d8e21ab0450e817c9bfd46cffc30] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp, totalSize=46.3 K 2024-11-25T12:58:23,602 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting ffb36df48ed6414ca6fbd91a664a65e8, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1732539499261 2024-11-25T12:58:23,602 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting dc25906560dd4aae9cd770ae83f35a89, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1732539501139 2024-11-25T12:58:23,603 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting fe2206c0c3694f9e99d891fa23d00bbd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732539501521 2024-11-25T12:58:23,603 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting e597d8e21ab0450e817c9bfd46cffc30, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1732539502179 2024-11-25T12:58:23,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742026_1202 (size=4469) 2024-11-25T12:58:23,610 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9806b0fa8f18a18b9b918dba8c42bc67#A#compaction#162 average throughput is 0.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:58:23,611 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/fd86c9b12ea74dc4a803c5350a0548b1 is 175, key is test_row_0/A:col10/1732539502181/Put/seqid=0 2024-11-25T12:58:23,626 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9806b0fa8f18a18b9b918dba8c42bc67#C#compaction#165 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:58:23,627 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/C/e202f60d2f3c4d3681774fb82461e272 is 50, key is test_row_0/C:col10/1732539502181/Put/seqid=0 2024-11-25T12:58:23,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742027_1203 (size=12454) 2024-11-25T12:58:23,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-25T12:58:23,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:23,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59968 deadline: 1732539563664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:23,667 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:23,667 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:23,667 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:23,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:23,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59966 deadline: 1732539563667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:23,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:23,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59944 deadline: 1732539563667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:23,671 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:23,671 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:23,671 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:23,671 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:23,671 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:23,671 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:23,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742028_1204 (size=32039) 2024-11-25T12:58:23,691 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/fd86c9b12ea74dc4a803c5350a0548b1 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/fd86c9b12ea74dc4a803c5350a0548b1 2024-11-25T12:58:23,698 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9806b0fa8f18a18b9b918dba8c42bc67/A of 9806b0fa8f18a18b9b918dba8c42bc67 into fd86c9b12ea74dc4a803c5350a0548b1(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:58:23,698 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9806b0fa8f18a18b9b918dba8c42bc67: 2024-11-25T12:58:23,698 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., storeName=9806b0fa8f18a18b9b918dba8c42bc67/A, priority=12, startTime=1732539503458; duration=0sec 2024-11-25T12:58:23,698 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:58:23,698 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9806b0fa8f18a18b9b918dba8c42bc67:A 2024-11-25T12:58:23,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742029_1205 (size=13085) 2024-11-25T12:58:23,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:23,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59968 deadline: 1732539563869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:23,873 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:23,873 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:23,873 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:23,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:23,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59944 deadline: 1732539563880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:23,884 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:23,885 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:23,885 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:23,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:23,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59966 deadline: 1732539563880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:23,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:23,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:23,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:24,041 INFO [Thread-574 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-574. 2024-11-25T12:58:24,041 DEBUG [Thread-574 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:58:24,041 DEBUG [Thread-574 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:58:24,041 DEBUG [Thread-574 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:58:24,041 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:58:24,043 INFO [Thread-578 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-578. 2024-11-25T12:58:24,043 DEBUG [Thread-578 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:58:24,043 DEBUG [Thread-578 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:58:24,043 DEBUG [Thread-578 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:58:24,043 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:58:24,044 INFO [Thread-576 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-576. 2024-11-25T12:58:24,045 DEBUG [Thread-576 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:58:24,045 DEBUG [Thread-576 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:58:24,045 DEBUG [Thread-576 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:58:24,045 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:58:24,047 INFO [Thread-580 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-580. 2024-11-25T12:58:24,047 DEBUG [Thread-580 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:58:24,047 DEBUG [Thread-580 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:58:24,047 DEBUG [Thread-580 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:58:24,047 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:58:24,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:24,066 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411253c3926abc0cb441a899219fc8d15ea62_9806b0fa8f18a18b9b918dba8c42bc67 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411253c3926abc0cb441a899219fc8d15ea62_9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:24,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/ccd49f64759745f3933e7b530ad025dc, store: [table=TestAcidGuarantees family=A region=9806b0fa8f18a18b9b918dba8c42bc67] 2024-11-25T12:58:24,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/ccd49f64759745f3933e7b530ad025dc is 175, key is test_row_0/A:col10/1732539502879/Put/seqid=0 2024-11-25T12:58:24,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742030_1206 (size=31255) 2024-11-25T12:58:24,119 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/C/e202f60d2f3c4d3681774fb82461e272 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/e202f60d2f3c4d3681774fb82461e272 2024-11-25T12:58:24,123 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9806b0fa8f18a18b9b918dba8c42bc67/C of 9806b0fa8f18a18b9b918dba8c42bc67 into e202f60d2f3c4d3681774fb82461e272(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:58:24,123 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9806b0fa8f18a18b9b918dba8c42bc67: 2024-11-25T12:58:24,123 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., storeName=9806b0fa8f18a18b9b918dba8c42bc67/C, priority=12, startTime=1732539503458; duration=0sec 2024-11-25T12:58:24,124 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:58:24,124 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9806b0fa8f18a18b9b918dba8c42bc67:C 2024-11-25T12:58:24,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-25T12:58:24,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:24,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59968 deadline: 1732539564179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:24,180 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:24,180 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:24,180 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:24,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:24,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:24,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59966 deadline: 1732539564190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:24,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59944 deadline: 1732539564190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:24,191 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:24,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:24,191 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:24,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:24,191 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:24,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:24,472 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=356, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/ccd49f64759745f3933e7b530ad025dc 2024-11-25T12:58:24,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/B/7a7791644e334e60977abc8c2e5dbe1b is 50, key is test_row_0/B:col10/1732539502879/Put/seqid=0 2024-11-25T12:58:24,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742031_1207 (size=12301) 2024-11-25T12:58:24,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:24,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59968 deadline: 1732539564690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:24,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:24,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:24,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:24,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:24,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:24,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59944 deadline: 1732539564700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:24,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59966 deadline: 1732539564700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:24,701 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:24,701 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:24,701 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:24,701 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:24,701 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:24,701 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:58:24,885 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/B/7a7791644e334e60977abc8c2e5dbe1b 2024-11-25T12:58:24,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/C/d0b061283710412d926cc530757ee3cf is 50, key is test_row_0/C:col10/1732539502879/Put/seqid=0 2024-11-25T12:58:24,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742032_1208 (size=12301) 2024-11-25T12:58:25,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-25T12:58:25,321 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/C/d0b061283710412d926cc530757ee3cf 2024-11-25T12:58:25,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/ccd49f64759745f3933e7b530ad025dc as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/ccd49f64759745f3933e7b530ad025dc 2024-11-25T12:58:25,331 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/ccd49f64759745f3933e7b530ad025dc, entries=150, sequenceid=356, filesize=30.5 K 2024-11-25T12:58:25,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/B/7a7791644e334e60977abc8c2e5dbe1b as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/7a7791644e334e60977abc8c2e5dbe1b 2024-11-25T12:58:25,337 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/7a7791644e334e60977abc8c2e5dbe1b, entries=150, sequenceid=356, filesize=12.0 K 2024-11-25T12:58:25,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/C/d0b061283710412d926cc530757ee3cf as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/d0b061283710412d926cc530757ee3cf 2024-11-25T12:58:25,343 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/d0b061283710412d926cc530757ee3cf, entries=150, sequenceid=356, filesize=12.0 K 2024-11-25T12:58:25,344 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 9806b0fa8f18a18b9b918dba8c42bc67 in 1841ms, sequenceid=356, compaction requested=false 2024-11-25T12:58:25,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 9806b0fa8f18a18b9b918dba8c42bc67: 2024-11-25T12:58:25,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:25,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-25T12:58:25,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-25T12:58:25,358 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-25T12:58:25,358 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3210 sec 2024-11-25T12:58:25,362 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 2.3280 sec 2024-11-25T12:58:25,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:25,701 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:58:25,702 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9806b0fa8f18a18b9b918dba8c42bc67 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-25T12:58:25,702 INFO [Thread-563 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-563. 2024-11-25T12:58:25,702 DEBUG [Thread-563 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:58:25,702 DEBUG [Thread-563 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:58:25,702 DEBUG [Thread-563 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:58:25,702 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:58:25,710 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411255b7db1d9d4764cd9861aec87768afe97_9806b0fa8f18a18b9b918dba8c42bc67 is 50, key is test_row_0/A:col10/1732539503543/Put/seqid=0 2024-11-25T12:58:25,712 INFO [Thread-569 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-569. 2024-11-25T12:58:25,712 DEBUG [Thread-569 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:58:25,712 DEBUG [Thread-569 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:58:25,712 INFO [Thread-567 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-567. 2024-11-25T12:58:25,712 DEBUG [Thread-569 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:58:25,712 DEBUG [Thread-567 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:58:25,712 DEBUG [Thread-567 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:58:25,712 DEBUG [Thread-567 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:58:25,712 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:58:25,712 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:58:25,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742033_1209 (size=12454) 2024-11-25T12:58:26,115 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:26,124 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411255b7db1d9d4764cd9861aec87768afe97_9806b0fa8f18a18b9b918dba8c42bc67 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411255b7db1d9d4764cd9861aec87768afe97_9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:26,125 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/4562566dda9047ddb259d0304fc4842e, store: [table=TestAcidGuarantees family=A region=9806b0fa8f18a18b9b918dba8c42bc67] 2024-11-25T12:58:26,126 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/4562566dda9047ddb259d0304fc4842e is 175, key is test_row_0/A:col10/1732539503543/Put/seqid=0 2024-11-25T12:58:26,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742034_1210 (size=31255) 2024-11-25T12:58:26,532 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=372, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/4562566dda9047ddb259d0304fc4842e 2024-11-25T12:58:26,540 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/B/7d839d7d094944d486dc36e5ce039ebe is 50, key is test_row_0/B:col10/1732539503543/Put/seqid=0 2024-11-25T12:58:26,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742035_1211 (size=12301) 2024-11-25T12:58:26,549 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/B/7d839d7d094944d486dc36e5ce039ebe 2024-11-25T12:58:26,574 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/C/0507f64b5c5f47bfa354a1f6582508e7 is 50, key is test_row_0/C:col10/1732539503543/Put/seqid=0 2024-11-25T12:58:26,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742036_1212 (size=12301) 2024-11-25T12:58:27,000 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/C/0507f64b5c5f47bfa354a1f6582508e7 2024-11-25T12:58:27,006 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/4562566dda9047ddb259d0304fc4842e as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/4562566dda9047ddb259d0304fc4842e 2024-11-25T12:58:27,011 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/4562566dda9047ddb259d0304fc4842e, entries=150, sequenceid=372, filesize=30.5 K 2024-11-25T12:58:27,011 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/B/7d839d7d094944d486dc36e5ce039ebe as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/7d839d7d094944d486dc36e5ce039ebe 2024-11-25T12:58:27,015 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/7d839d7d094944d486dc36e5ce039ebe, entries=150, sequenceid=372, filesize=12.0 K 2024-11-25T12:58:27,016 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/C/0507f64b5c5f47bfa354a1f6582508e7 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/0507f64b5c5f47bfa354a1f6582508e7 2024-11-25T12:58:27,020 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/0507f64b5c5f47bfa354a1f6582508e7, entries=150, sequenceid=372, filesize=12.0 K 2024-11-25T12:58:27,021 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=13.42 KB/13740 for 9806b0fa8f18a18b9b918dba8c42bc67 in 1320ms, sequenceid=372, compaction requested=true 2024-11-25T12:58:27,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9806b0fa8f18a18b9b918dba8c42bc67: 2024-11-25T12:58:27,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9806b0fa8f18a18b9b918dba8c42bc67:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:58:27,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:58:27,022 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:58:27,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9806b0fa8f18a18b9b918dba8c42bc67:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:58:27,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:58:27,022 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:58:27,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9806b0fa8f18a18b9b918dba8c42bc67:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:58:27,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:58:27,023 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94549 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:58:27,023 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:58:27,023 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1541): 9806b0fa8f18a18b9b918dba8c42bc67/A is initiating minor compaction (all files) 2024-11-25T12:58:27,023 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): 9806b0fa8f18a18b9b918dba8c42bc67/B is initiating minor compaction (all files) 2024-11-25T12:58:27,023 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9806b0fa8f18a18b9b918dba8c42bc67/A in TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:27,023 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9806b0fa8f18a18b9b918dba8c42bc67/B in TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:27,023 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/fd86c9b12ea74dc4a803c5350a0548b1, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/ccd49f64759745f3933e7b530ad025dc, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/4562566dda9047ddb259d0304fc4842e] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp, totalSize=92.3 K 2024-11-25T12:58:27,023 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/4f907b83f8be4beea6cc888f6b12a0d9, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/7a7791644e334e60977abc8c2e5dbe1b, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/7d839d7d094944d486dc36e5ce039ebe] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp, totalSize=36.8 K 2024-11-25T12:58:27,023 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:27,023 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. files: [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/fd86c9b12ea74dc4a803c5350a0548b1, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/ccd49f64759745f3933e7b530ad025dc, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/4562566dda9047ddb259d0304fc4842e] 2024-11-25T12:58:27,023 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f907b83f8be4beea6cc888f6b12a0d9, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1732539502179 2024-11-25T12:58:27,024 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd86c9b12ea74dc4a803c5350a0548b1, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1732539502179 2024-11-25T12:58:27,024 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a7791644e334e60977abc8c2e5dbe1b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1732539502873 2024-11-25T12:58:27,024 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting ccd49f64759745f3933e7b530ad025dc, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1732539502873 2024-11-25T12:58:27,024 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d839d7d094944d486dc36e5ce039ebe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1732539503536 2024-11-25T12:58:27,024 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4562566dda9047ddb259d0304fc4842e, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1732539503536 2024-11-25T12:58:27,033 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9806b0fa8f18a18b9b918dba8c42bc67#B#compaction#171 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:58:27,033 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/B/9cc0236043ad4e54b121d42f727266c6 is 50, key is test_row_0/B:col10/1732539503543/Put/seqid=0 2024-11-25T12:58:27,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742037_1213 (size=13187) 2024-11-25T12:58:27,047 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9806b0fa8f18a18b9b918dba8c42bc67] 2024-11-25T12:58:27,056 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/B/9cc0236043ad4e54b121d42f727266c6 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/9cc0236043ad4e54b121d42f727266c6 2024-11-25T12:58:27,068 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241125cabc74c0ea8a4b8cac1f94c0d5821693_9806b0fa8f18a18b9b918dba8c42bc67 store=[table=TestAcidGuarantees family=A region=9806b0fa8f18a18b9b918dba8c42bc67] 2024-11-25T12:58:27,077 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9806b0fa8f18a18b9b918dba8c42bc67/B of 9806b0fa8f18a18b9b918dba8c42bc67 into 9cc0236043ad4e54b121d42f727266c6(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:58:27,077 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9806b0fa8f18a18b9b918dba8c42bc67: 2024-11-25T12:58:27,077 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., storeName=9806b0fa8f18a18b9b918dba8c42bc67/B, priority=13, startTime=1732539507022; duration=0sec 2024-11-25T12:58:27,078 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:58:27,078 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9806b0fa8f18a18b9b918dba8c42bc67:B 2024-11-25T12:58:27,078 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:58:27,079 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:58:27,079 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): 9806b0fa8f18a18b9b918dba8c42bc67/C is initiating minor compaction (all files) 2024-11-25T12:58:27,079 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9806b0fa8f18a18b9b918dba8c42bc67/C in TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:27,079 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/e202f60d2f3c4d3681774fb82461e272, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/d0b061283710412d926cc530757ee3cf, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/0507f64b5c5f47bfa354a1f6582508e7] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp, totalSize=36.8 K 2024-11-25T12:58:27,079 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting e202f60d2f3c4d3681774fb82461e272, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1732539502179 2024-11-25T12:58:27,080 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting d0b061283710412d926cc530757ee3cf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1732539502873 2024-11-25T12:58:27,080 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 0507f64b5c5f47bfa354a1f6582508e7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1732539503536 2024-11-25T12:58:27,111 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241125cabc74c0ea8a4b8cac1f94c0d5821693_9806b0fa8f18a18b9b918dba8c42bc67, store=[table=TestAcidGuarantees family=A region=9806b0fa8f18a18b9b918dba8c42bc67] 2024-11-25T12:58:27,111 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125cabc74c0ea8a4b8cac1f94c0d5821693_9806b0fa8f18a18b9b918dba8c42bc67 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9806b0fa8f18a18b9b918dba8c42bc67] 2024-11-25T12:58:27,138 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9806b0fa8f18a18b9b918dba8c42bc67#C#compaction#173 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:58:27,138 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/C/5831f42ef2584de9a284abfee088aac1 is 50, key is test_row_0/C:col10/1732539503543/Put/seqid=0 2024-11-25T12:58:27,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742038_1214 (size=4469) 2024-11-25T12:58:27,156 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9806b0fa8f18a18b9b918dba8c42bc67#A#compaction#172 average throughput is 0.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:58:27,157 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/5c3265739cae42c5a23ffee187f497a5 is 175, key is test_row_0/A:col10/1732539503543/Put/seqid=0 2024-11-25T12:58:27,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-25T12:58:27,192 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:58:27,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742040_1216 (size=32141) 2024-11-25T12:58:27,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742039_1215 (size=13187) 2024-11-25T12:58:27,355 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-25T12:58:27,597 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/5c3265739cae42c5a23ffee187f497a5 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/5c3265739cae42c5a23ffee187f497a5 2024-11-25T12:58:27,599 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/C/5831f42ef2584de9a284abfee088aac1 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/5831f42ef2584de9a284abfee088aac1 2024-11-25T12:58:27,602 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9806b0fa8f18a18b9b918dba8c42bc67/A of 9806b0fa8f18a18b9b918dba8c42bc67 into 5c3265739cae42c5a23ffee187f497a5(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:58:27,602 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9806b0fa8f18a18b9b918dba8c42bc67: 2024-11-25T12:58:27,602 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., storeName=9806b0fa8f18a18b9b918dba8c42bc67/A, priority=13, startTime=1732539507021; duration=0sec 2024-11-25T12:58:27,602 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:58:27,602 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9806b0fa8f18a18b9b918dba8c42bc67:A 2024-11-25T12:58:27,604 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9806b0fa8f18a18b9b918dba8c42bc67/C of 9806b0fa8f18a18b9b918dba8c42bc67 into 5831f42ef2584de9a284abfee088aac1(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:58:27,604 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9806b0fa8f18a18b9b918dba8c42bc67: 2024-11-25T12:58:27,604 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., storeName=9806b0fa8f18a18b9b918dba8c42bc67/C, priority=13, startTime=1732539507022; duration=0sec 2024-11-25T12:58:27,604 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:58:27,604 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9806b0fa8f18a18b9b918dba8c42bc67:C 2024-11-25T12:58:31,573 INFO [Thread-565 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-565. 2024-11-25T12:58:31,573 DEBUG [Thread-565 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:58:31,573 DEBUG [Thread-565 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:58:31,573 DEBUG [Thread-565 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:58:31,573 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:58:31,581 INFO [Thread-571 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-571. 2024-11-25T12:58:31,582 DEBUG [Thread-571 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:58:31,582 DEBUG [Thread-571 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:58:31,582 DEBUG [Thread-571 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:58:31,582 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:58:31,582 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-25T12:58:31,582 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 45 2024-11-25T12:58:31,582 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 35 2024-11-25T12:58:31,582 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 121 2024-11-25T12:58:31,582 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 56 2024-11-25T12:58:31,582 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 37 2024-11-25T12:58:31,582 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-25T12:58:31,582 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2492 2024-11-25T12:58:31,582 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2341 2024-11-25T12:58:31,582 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-25T12:58:31,582 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1159 2024-11-25T12:58:31,582 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3471 rows 2024-11-25T12:58:31,583 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1136 2024-11-25T12:58:31,583 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3408 rows 2024-11-25T12:58:31,583 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T12:58:31,583 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T12:58:31,583 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:58:31,584 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:58:31,584 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:58:31,585 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-25T12:58:31,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-25T12:58:31,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-25T12:58:31,589 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732539511588"}]},"ts":"1732539511588"} 2024-11-25T12:58:31,590 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-25T12:58:31,590 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-25T12:58:31,591 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-25T12:58:31,592 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9806b0fa8f18a18b9b918dba8c42bc67, UNASSIGN}] 2024-11-25T12:58:31,593 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9806b0fa8f18a18b9b918dba8c42bc67, UNASSIGN 2024-11-25T12:58:31,593 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=52 updating hbase:meta row=9806b0fa8f18a18b9b918dba8c42bc67, regionState=CLOSING, regionLocation=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:31,595 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9806b0fa8f18a18b9b918dba8c42bc67, UNASSIGN because future has completed 2024-11-25T12:58:31,595 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-25T12:58:31,595 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246}] 2024-11-25T12:58:31,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-25T12:58:31,746 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] handler.UnassignRegionHandler(122): Close 9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:31,747 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-25T12:58:31,747 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1722): Closing 9806b0fa8f18a18b9b918dba8c42bc67, disabling compactions & flushes 2024-11-25T12:58:31,747 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:31,747 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:31,747 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. after waiting 0 ms 2024-11-25T12:58:31,747 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:31,747 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(2902): Flushing 9806b0fa8f18a18b9b918dba8c42bc67 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-25T12:58:31,753 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112528e5f5c37307491683ca06aa77f26cf1_9806b0fa8f18a18b9b918dba8c42bc67 is 50, key is test_row_0/A:col10/1732539511571/Put/seqid=0 2024-11-25T12:58:31,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742041_1217 (size=9914) 2024-11-25T12:58:31,758 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:31,765 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112528e5f5c37307491683ca06aa77f26cf1_9806b0fa8f18a18b9b918dba8c42bc67 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112528e5f5c37307491683ca06aa77f26cf1_9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:31,766 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/432199f8d867437bad90c2e46d13bcc6, store: [table=TestAcidGuarantees family=A region=9806b0fa8f18a18b9b918dba8c42bc67] 2024-11-25T12:58:31,766 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/432199f8d867437bad90c2e46d13bcc6 is 175, key is test_row_0/A:col10/1732539511571/Put/seqid=0 2024-11-25T12:58:31,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742042_1218 (size=22561) 2024-11-25T12:58:31,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-25T12:58:32,180 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=382, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/432199f8d867437bad90c2e46d13bcc6 2024-11-25T12:58:32,188 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/B/286055dfb50e47a4ab04677fe35c1c9d is 50, key is test_row_0/B:col10/1732539511571/Put/seqid=0 2024-11-25T12:58:32,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742043_1219 (size=9857) 2024-11-25T12:58:32,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-25T12:58:32,592 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/B/286055dfb50e47a4ab04677fe35c1c9d 2024-11-25T12:58:32,628 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/C/65b3fda9596c4237aa4c8731f2846dd6 is 50, key is test_row_0/C:col10/1732539511571/Put/seqid=0 2024-11-25T12:58:32,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742044_1220 (size=9857) 2024-11-25T12:58:32,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-25T12:58:33,048 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/C/65b3fda9596c4237aa4c8731f2846dd6 2024-11-25T12:58:33,052 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/A/432199f8d867437bad90c2e46d13bcc6 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/432199f8d867437bad90c2e46d13bcc6 2024-11-25T12:58:33,056 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/432199f8d867437bad90c2e46d13bcc6, entries=100, sequenceid=382, filesize=22.0 K 2024-11-25T12:58:33,057 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/B/286055dfb50e47a4ab04677fe35c1c9d as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/286055dfb50e47a4ab04677fe35c1c9d 2024-11-25T12:58:33,060 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/286055dfb50e47a4ab04677fe35c1c9d, entries=100, sequenceid=382, filesize=9.6 K 2024-11-25T12:58:33,061 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/.tmp/C/65b3fda9596c4237aa4c8731f2846dd6 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/65b3fda9596c4237aa4c8731f2846dd6 2024-11-25T12:58:33,064 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/65b3fda9596c4237aa4c8731f2846dd6, entries=100, sequenceid=382, filesize=9.6 K 2024-11-25T12:58:33,065 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 9806b0fa8f18a18b9b918dba8c42bc67 in 1318ms, sequenceid=382, compaction requested=false 2024-11-25T12:58:33,065 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/9bf62f7391f440abb5089020f45c6f80, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/42deca4a7a7a443b8807f66c48a02425, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/76274d615dc649169b1980a5d941b129, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/45b4db3b2f954395ac6a5ad67fcbed6e, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/a4e1a73720dd40e8a3fe8d17add0b9ee, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/48f0f15698d041a0b35a39bcdfae5ae0, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/3ba02a1801164515b68ae50df492f529, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/d3eee17f68454bc2814e07605f204184, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/708de6d9542746ef9414786c61c26e05, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/b9d70a8295d946d38fdd8d9815fa2fee, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/2da1382ddb824217880c1e6f67c54369, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/09702298e36948539ba0a20f52945123, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/620ace6715e54c00abfa1de3f1407cf8, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/ed64e7db165e42c9a08986295b673456, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/ef98c514cf7d408aad8ca10600c819a7, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/ef4a7356df454630925a123a5fdcd90c, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/a99855dc5ceb4a5680df06abce0434b0, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/27c4f6483e414fc198832c96dbf8d042, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/a13208199b614024a57666c84e2eeff8, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/cb8a727d82904cb9b6989ab1ad670885, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/3600d5198fa84231aa24778d39f97a68, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/2692d92740e440caa6a5bf96faec6c70, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/fd86c9b12ea74dc4a803c5350a0548b1, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/4e218e975942401cb4fac3c4558efa20, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/ccd49f64759745f3933e7b530ad025dc, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/4562566dda9047ddb259d0304fc4842e] to archive 2024-11-25T12:58:33,066 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T12:58:33,068 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/9bf62f7391f440abb5089020f45c6f80 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/9bf62f7391f440abb5089020f45c6f80 2024-11-25T12:58:33,069 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/42deca4a7a7a443b8807f66c48a02425 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/42deca4a7a7a443b8807f66c48a02425 2024-11-25T12:58:33,070 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/76274d615dc649169b1980a5d941b129 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/76274d615dc649169b1980a5d941b129 2024-11-25T12:58:33,071 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/45b4db3b2f954395ac6a5ad67fcbed6e to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/45b4db3b2f954395ac6a5ad67fcbed6e 2024-11-25T12:58:33,071 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/a4e1a73720dd40e8a3fe8d17add0b9ee to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/a4e1a73720dd40e8a3fe8d17add0b9ee 2024-11-25T12:58:33,072 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/48f0f15698d041a0b35a39bcdfae5ae0 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/48f0f15698d041a0b35a39bcdfae5ae0 2024-11-25T12:58:33,073 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/3ba02a1801164515b68ae50df492f529 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/3ba02a1801164515b68ae50df492f529 2024-11-25T12:58:33,074 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/d3eee17f68454bc2814e07605f204184 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/d3eee17f68454bc2814e07605f204184 2024-11-25T12:58:33,075 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/708de6d9542746ef9414786c61c26e05 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/708de6d9542746ef9414786c61c26e05 2024-11-25T12:58:33,076 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/b9d70a8295d946d38fdd8d9815fa2fee to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/b9d70a8295d946d38fdd8d9815fa2fee 2024-11-25T12:58:33,076 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/2da1382ddb824217880c1e6f67c54369 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/2da1382ddb824217880c1e6f67c54369 2024-11-25T12:58:33,077 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/09702298e36948539ba0a20f52945123 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/09702298e36948539ba0a20f52945123 2024-11-25T12:58:33,078 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/620ace6715e54c00abfa1de3f1407cf8 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/620ace6715e54c00abfa1de3f1407cf8 2024-11-25T12:58:33,079 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/ed64e7db165e42c9a08986295b673456 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/ed64e7db165e42c9a08986295b673456 2024-11-25T12:58:33,080 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/ef98c514cf7d408aad8ca10600c819a7 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/ef98c514cf7d408aad8ca10600c819a7 2024-11-25T12:58:33,081 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/ef4a7356df454630925a123a5fdcd90c to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/ef4a7356df454630925a123a5fdcd90c 2024-11-25T12:58:33,082 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/a99855dc5ceb4a5680df06abce0434b0 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/a99855dc5ceb4a5680df06abce0434b0 2024-11-25T12:58:33,083 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/27c4f6483e414fc198832c96dbf8d042 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/27c4f6483e414fc198832c96dbf8d042 2024-11-25T12:58:33,083 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/a13208199b614024a57666c84e2eeff8 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/a13208199b614024a57666c84e2eeff8 2024-11-25T12:58:33,084 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/cb8a727d82904cb9b6989ab1ad670885 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/cb8a727d82904cb9b6989ab1ad670885 2024-11-25T12:58:33,085 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/3600d5198fa84231aa24778d39f97a68 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/3600d5198fa84231aa24778d39f97a68 2024-11-25T12:58:33,086 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/2692d92740e440caa6a5bf96faec6c70 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/2692d92740e440caa6a5bf96faec6c70 2024-11-25T12:58:33,087 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/fd86c9b12ea74dc4a803c5350a0548b1 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/fd86c9b12ea74dc4a803c5350a0548b1 2024-11-25T12:58:33,087 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/4e218e975942401cb4fac3c4558efa20 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/4e218e975942401cb4fac3c4558efa20 2024-11-25T12:58:33,088 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/ccd49f64759745f3933e7b530ad025dc to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/ccd49f64759745f3933e7b530ad025dc 2024-11-25T12:58:33,089 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/4562566dda9047ddb259d0304fc4842e to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/4562566dda9047ddb259d0304fc4842e 2024-11-25T12:58:33,091 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/5f4a1ef9f4eb42c08ad186f6bcac23c7, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/50b421bbb831456bafdf501a2a314b19, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/abb9530cb5324d9fb6e927527e32e0e1, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/7f273fcdf56341b3af1ae9a8a1851e64, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/021b38eb770f4bffb029885ed7a91a82, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/40a49f11b38d4dcf87eb05631338c71a, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/d6f9df5c04104e5b9b4016160c7f9432, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/b73d79d4de1c4c7f9057aaf4adf92424, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/5f22d2daace04d9c9e56783b4b996c0e, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/cec2ac5785114183babe8dda33cabcb8, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/9977f799cb0a4fd68d5f97914d5334be, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/f1ba489c109f49b382b0f6d0347469c5, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/daf2b242c2564b7380145d0d5553c1e1, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/79300711dbf144e29559ec96e5759daa, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/6ad51c8db5614496be55d3b39f0ea3ad, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/ff4d53f662fb4bebb69da3f19abf750d, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/d0c4797d673548d7b467573f636f9a78, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/2c6851f994654fe08d69e38b79755a00, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/6b4c129c0e2644869731b7a5f8885904, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/4c5306706dbb4a9e8c9c104145f2bf2d, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/720795ad923e44acbf5828a6440b9f67, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/ae30b18ed26242edb7c44fe8fcf1f035, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/4f907b83f8be4beea6cc888f6b12a0d9, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/77c4c2b4d9414cf5aaab19403845fff7, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/7a7791644e334e60977abc8c2e5dbe1b, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/7d839d7d094944d486dc36e5ce039ebe] to archive 2024-11-25T12:58:33,091 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T12:58:33,093 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/5f4a1ef9f4eb42c08ad186f6bcac23c7 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/5f4a1ef9f4eb42c08ad186f6bcac23c7 2024-11-25T12:58:33,094 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/50b421bbb831456bafdf501a2a314b19 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/50b421bbb831456bafdf501a2a314b19 2024-11-25T12:58:33,095 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/abb9530cb5324d9fb6e927527e32e0e1 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/abb9530cb5324d9fb6e927527e32e0e1 2024-11-25T12:58:33,096 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/7f273fcdf56341b3af1ae9a8a1851e64 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/7f273fcdf56341b3af1ae9a8a1851e64 2024-11-25T12:58:33,097 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/021b38eb770f4bffb029885ed7a91a82 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/021b38eb770f4bffb029885ed7a91a82 2024-11-25T12:58:33,098 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/40a49f11b38d4dcf87eb05631338c71a to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/40a49f11b38d4dcf87eb05631338c71a 2024-11-25T12:58:33,099 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/d6f9df5c04104e5b9b4016160c7f9432 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/d6f9df5c04104e5b9b4016160c7f9432 2024-11-25T12:58:33,100 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/b73d79d4de1c4c7f9057aaf4adf92424 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/b73d79d4de1c4c7f9057aaf4adf92424 2024-11-25T12:58:33,101 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/5f22d2daace04d9c9e56783b4b996c0e to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/5f22d2daace04d9c9e56783b4b996c0e 2024-11-25T12:58:33,102 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/cec2ac5785114183babe8dda33cabcb8 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/cec2ac5785114183babe8dda33cabcb8 2024-11-25T12:58:33,102 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/9977f799cb0a4fd68d5f97914d5334be to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/9977f799cb0a4fd68d5f97914d5334be 2024-11-25T12:58:33,103 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/f1ba489c109f49b382b0f6d0347469c5 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/f1ba489c109f49b382b0f6d0347469c5 2024-11-25T12:58:33,104 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/daf2b242c2564b7380145d0d5553c1e1 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/daf2b242c2564b7380145d0d5553c1e1 2024-11-25T12:58:33,105 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/79300711dbf144e29559ec96e5759daa to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/79300711dbf144e29559ec96e5759daa 2024-11-25T12:58:33,106 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/6ad51c8db5614496be55d3b39f0ea3ad to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/6ad51c8db5614496be55d3b39f0ea3ad 2024-11-25T12:58:33,107 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/ff4d53f662fb4bebb69da3f19abf750d to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/ff4d53f662fb4bebb69da3f19abf750d 2024-11-25T12:58:33,108 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/d0c4797d673548d7b467573f636f9a78 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/d0c4797d673548d7b467573f636f9a78 2024-11-25T12:58:33,109 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/2c6851f994654fe08d69e38b79755a00 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/2c6851f994654fe08d69e38b79755a00 2024-11-25T12:58:33,109 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/6b4c129c0e2644869731b7a5f8885904 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/6b4c129c0e2644869731b7a5f8885904 2024-11-25T12:58:33,110 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/4c5306706dbb4a9e8c9c104145f2bf2d to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/4c5306706dbb4a9e8c9c104145f2bf2d 2024-11-25T12:58:33,111 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/720795ad923e44acbf5828a6440b9f67 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/720795ad923e44acbf5828a6440b9f67 2024-11-25T12:58:33,112 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/ae30b18ed26242edb7c44fe8fcf1f035 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/ae30b18ed26242edb7c44fe8fcf1f035 2024-11-25T12:58:33,113 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/4f907b83f8be4beea6cc888f6b12a0d9 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/4f907b83f8be4beea6cc888f6b12a0d9 2024-11-25T12:58:33,114 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/77c4c2b4d9414cf5aaab19403845fff7 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/77c4c2b4d9414cf5aaab19403845fff7 2024-11-25T12:58:33,115 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/7a7791644e334e60977abc8c2e5dbe1b to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/7a7791644e334e60977abc8c2e5dbe1b 2024-11-25T12:58:33,116 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/7d839d7d094944d486dc36e5ce039ebe to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/7d839d7d094944d486dc36e5ce039ebe 2024-11-25T12:58:33,118 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/1545a41de1ce4180a663b971f0436b02, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/782d48a2df4945caa79383ba393df85a, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/d240fdce83f94459be2a8f384cf3dc9d, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/8e9fc41ed9454d60ab1ed6690447bb77, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/19973b88a14e4e17ab77d2f88fd60538, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/35804b79e49146b58ebfcd1146434bb5, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/d5674cb220384b95b9be680e38c1f2ba, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/046ce0ad39ae48eaa1206d3508685b7e, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/2d4a1f40c93c4ca59c478dff3b8b0776, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/e113ca4f6454470892132618e44b756e, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/5b8afe89241a411e8ba40e16f77ecc66, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/3ba99245a9484ed3845e735e31e9efa5, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/43140d42e3454be4aaee229c3ed44be5, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/86fa443247b14e229df3cecdf466d69d, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/14bb6e453bda4ccaa2725b43d8b15427, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/8663bc1835b14073a2ee92d3f99e1da6, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/ac39d140c1ec4411ad31dd826e688ded, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/d459d7f867db4fcb8b60b8b52c210053, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/ffb36df48ed6414ca6fbd91a664a65e8, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/b62e0ea79cb4430889f80c36d187ae59, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/dc25906560dd4aae9cd770ae83f35a89, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/fe2206c0c3694f9e99d891fa23d00bbd, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/e202f60d2f3c4d3681774fb82461e272, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/e597d8e21ab0450e817c9bfd46cffc30, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/d0b061283710412d926cc530757ee3cf, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/0507f64b5c5f47bfa354a1f6582508e7] to archive 2024-11-25T12:58:33,119 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T12:58:33,120 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/1545a41de1ce4180a663b971f0436b02 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/1545a41de1ce4180a663b971f0436b02 2024-11-25T12:58:33,121 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/782d48a2df4945caa79383ba393df85a to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/782d48a2df4945caa79383ba393df85a 2024-11-25T12:58:33,122 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/d240fdce83f94459be2a8f384cf3dc9d to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/d240fdce83f94459be2a8f384cf3dc9d 2024-11-25T12:58:33,123 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/8e9fc41ed9454d60ab1ed6690447bb77 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/8e9fc41ed9454d60ab1ed6690447bb77 2024-11-25T12:58:33,124 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/19973b88a14e4e17ab77d2f88fd60538 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/19973b88a14e4e17ab77d2f88fd60538 2024-11-25T12:58:33,125 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/35804b79e49146b58ebfcd1146434bb5 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/35804b79e49146b58ebfcd1146434bb5 2024-11-25T12:58:33,126 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/d5674cb220384b95b9be680e38c1f2ba to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/d5674cb220384b95b9be680e38c1f2ba 2024-11-25T12:58:33,127 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/046ce0ad39ae48eaa1206d3508685b7e to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/046ce0ad39ae48eaa1206d3508685b7e 2024-11-25T12:58:33,128 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/2d4a1f40c93c4ca59c478dff3b8b0776 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/2d4a1f40c93c4ca59c478dff3b8b0776 2024-11-25T12:58:33,128 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/e113ca4f6454470892132618e44b756e to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/e113ca4f6454470892132618e44b756e 2024-11-25T12:58:33,129 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/5b8afe89241a411e8ba40e16f77ecc66 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/5b8afe89241a411e8ba40e16f77ecc66 2024-11-25T12:58:33,130 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/3ba99245a9484ed3845e735e31e9efa5 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/3ba99245a9484ed3845e735e31e9efa5 2024-11-25T12:58:33,131 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/43140d42e3454be4aaee229c3ed44be5 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/43140d42e3454be4aaee229c3ed44be5 2024-11-25T12:58:33,132 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/86fa443247b14e229df3cecdf466d69d to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/86fa443247b14e229df3cecdf466d69d 2024-11-25T12:58:33,133 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/14bb6e453bda4ccaa2725b43d8b15427 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/14bb6e453bda4ccaa2725b43d8b15427 2024-11-25T12:58:33,134 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/8663bc1835b14073a2ee92d3f99e1da6 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/8663bc1835b14073a2ee92d3f99e1da6 2024-11-25T12:58:33,135 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/ac39d140c1ec4411ad31dd826e688ded to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/ac39d140c1ec4411ad31dd826e688ded 2024-11-25T12:58:33,136 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/d459d7f867db4fcb8b60b8b52c210053 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/d459d7f867db4fcb8b60b8b52c210053 2024-11-25T12:58:33,137 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/ffb36df48ed6414ca6fbd91a664a65e8 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/ffb36df48ed6414ca6fbd91a664a65e8 2024-11-25T12:58:33,138 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/b62e0ea79cb4430889f80c36d187ae59 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/b62e0ea79cb4430889f80c36d187ae59 2024-11-25T12:58:33,139 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/dc25906560dd4aae9cd770ae83f35a89 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/dc25906560dd4aae9cd770ae83f35a89 2024-11-25T12:58:33,140 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/fe2206c0c3694f9e99d891fa23d00bbd to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/fe2206c0c3694f9e99d891fa23d00bbd 2024-11-25T12:58:33,141 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/e202f60d2f3c4d3681774fb82461e272 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/e202f60d2f3c4d3681774fb82461e272 2024-11-25T12:58:33,141 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/e597d8e21ab0450e817c9bfd46cffc30 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/e597d8e21ab0450e817c9bfd46cffc30 2024-11-25T12:58:33,142 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/d0b061283710412d926cc530757ee3cf to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/d0b061283710412d926cc530757ee3cf 2024-11-25T12:58:33,143 DEBUG [StoreCloser-TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/0507f64b5c5f47bfa354a1f6582508e7 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/0507f64b5c5f47bfa354a1f6582508e7 2024-11-25T12:58:33,147 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/recovered.edits/385.seqid, newMaxSeqId=385, maxSeqId=4 2024-11-25T12:58:33,147 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. 2024-11-25T12:58:33,148 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1676): Region close journal for 9806b0fa8f18a18b9b918dba8c42bc67: Waiting for close lock at 1732539511747Running coprocessor pre-close hooks at 1732539511747Disabling compacts and flushes for region at 1732539511747Disabling writes for close at 1732539511747Obtaining lock to block concurrent updates at 1732539511747Preparing flush snapshotting stores in 9806b0fa8f18a18b9b918dba8c42bc67 at 1732539511747Finished memstore snapshotting TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1732539511747Flushing stores of TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67. at 1732539511748 (+1 ms)Flushing 9806b0fa8f18a18b9b918dba8c42bc67/A: creating writer at 1732539511748Flushing mob file 9806b0fa8f18a18b9b918dba8c42bc67/A: appending metadata at 1732539511753 (+5 ms)Flushing mob file 9806b0fa8f18a18b9b918dba8c42bc67/A: closing flushed file at 1732539511753Flushing 9806b0fa8f18a18b9b918dba8c42bc67/A: appending metadata at 1732539511766 (+13 ms)Flushing 9806b0fa8f18a18b9b918dba8c42bc67/A: closing flushed file at 1732539511766Flushing 9806b0fa8f18a18b9b918dba8c42bc67/B: creating writer at 1732539512185 (+419 ms)Flushing 9806b0fa8f18a18b9b918dba8c42bc67/B: appending metadata at 1732539512187 (+2 ms)Flushing 9806b0fa8f18a18b9b918dba8c42bc67/B: closing flushed file at 1732539512187Flushing 9806b0fa8f18a18b9b918dba8c42bc67/C: creating writer at 1732539512596 (+409 ms)Flushing 9806b0fa8f18a18b9b918dba8c42bc67/C: appending metadata at 1732539512599 (+3 ms)Flushing 9806b0fa8f18a18b9b918dba8c42bc67/C: closing flushed file at 1732539512599Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5b7e3156: reopening flushed file at 1732539513052 (+453 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@23d56793: reopening flushed file at 1732539513056 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@11296a1c: reopening flushed file at 1732539513060 (+4 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 9806b0fa8f18a18b9b918dba8c42bc67 in 1318ms, sequenceid=382, compaction requested=false at 1732539513065 (+5 ms)Writing region close event to WAL at 1732539513144 (+79 ms)Running coprocessor post-close hooks at 1732539513147 (+3 ms)Closed at 1732539513147 2024-11-25T12:58:33,149 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] handler.UnassignRegionHandler(157): Closed 9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:33,150 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=52 updating hbase:meta row=9806b0fa8f18a18b9b918dba8c42bc67, regionState=CLOSED 2024-11-25T12:58:33,151 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=52, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 because future has completed 2024-11-25T12:58:33,153 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-25T12:58:33,153 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; CloseRegionProcedure 9806b0fa8f18a18b9b918dba8c42bc67, server=1ff8009e463e,35905,1732539449246 in 1.5560 sec 2024-11-25T12:58:33,154 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=51 2024-11-25T12:58:33,154 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=51, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9806b0fa8f18a18b9b918dba8c42bc67, UNASSIGN in 1.5610 sec 2024-11-25T12:58:33,156 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-25T12:58:33,156 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5630 sec 2024-11-25T12:58:33,157 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732539513157"}]},"ts":"1732539513157"} 2024-11-25T12:58:33,158 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-25T12:58:33,158 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-25T12:58:33,159 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5730 sec 2024-11-25T12:58:33,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-25T12:58:33,740 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:58:33,741 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-25T12:58:33,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T12:58:33,743 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=54, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T12:58:33,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-25T12:58:33,744 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=54, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T12:58:33,767 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:33,771 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A, FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B, FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C, FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/recovered.edits] 2024-11-25T12:58:33,774 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/432199f8d867437bad90c2e46d13bcc6 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/432199f8d867437bad90c2e46d13bcc6 2024-11-25T12:58:33,775 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/5c3265739cae42c5a23ffee187f497a5 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/A/5c3265739cae42c5a23ffee187f497a5 2024-11-25T12:58:33,777 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/286055dfb50e47a4ab04677fe35c1c9d to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/286055dfb50e47a4ab04677fe35c1c9d 2024-11-25T12:58:33,778 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/9cc0236043ad4e54b121d42f727266c6 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/B/9cc0236043ad4e54b121d42f727266c6 2024-11-25T12:58:33,780 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/5831f42ef2584de9a284abfee088aac1 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/5831f42ef2584de9a284abfee088aac1 2024-11-25T12:58:33,781 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/65b3fda9596c4237aa4c8731f2846dd6 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/C/65b3fda9596c4237aa4c8731f2846dd6 2024-11-25T12:58:33,783 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/recovered.edits/385.seqid to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67/recovered.edits/385.seqid 2024-11-25T12:58:33,784 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:33,784 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-25T12:58:33,785 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-25T12:58:33,785 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-25T12:58:33,789 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125079e2a32619647739a5cb9c6a8b83e83_9806b0fa8f18a18b9b918dba8c42bc67 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125079e2a32619647739a5cb9c6a8b83e83_9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:33,790 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112518b243159d494ae983b9012acf8fb9bb_9806b0fa8f18a18b9b918dba8c42bc67 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112518b243159d494ae983b9012acf8fb9bb_9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:33,791 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112528e5f5c37307491683ca06aa77f26cf1_9806b0fa8f18a18b9b918dba8c42bc67 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112528e5f5c37307491683ca06aa77f26cf1_9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:33,792 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411252d4828d17d6b4d0d99526ee70a6f92cf_9806b0fa8f18a18b9b918dba8c42bc67 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411252d4828d17d6b4d0d99526ee70a6f92cf_9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:33,793 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411253641b6e4a394435689092a05f886538e_9806b0fa8f18a18b9b918dba8c42bc67 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411253641b6e4a394435689092a05f886538e_9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:33,794 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411253c3926abc0cb441a899219fc8d15ea62_9806b0fa8f18a18b9b918dba8c42bc67 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411253c3926abc0cb441a899219fc8d15ea62_9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:33,795 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411253ca5aaf3761f4bb4a946505051d92581_9806b0fa8f18a18b9b918dba8c42bc67 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411253ca5aaf3761f4bb4a946505051d92581_9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:33,796 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411254e6c47e9a4be405891be7b6b96c0b272_9806b0fa8f18a18b9b918dba8c42bc67 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411254e6c47e9a4be405891be7b6b96c0b272_9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:33,797 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411255b7db1d9d4764cd9861aec87768afe97_9806b0fa8f18a18b9b918dba8c42bc67 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411255b7db1d9d4764cd9861aec87768afe97_9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:33,798 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411256849d301e9fa48dea0b414779257cc40_9806b0fa8f18a18b9b918dba8c42bc67 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411256849d301e9fa48dea0b414779257cc40_9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:33,798 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112582f6b93d490048bcaad54b78ba8095c0_9806b0fa8f18a18b9b918dba8c42bc67 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112582f6b93d490048bcaad54b78ba8095c0_9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:33,800 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411258bcc9d8f8f704f318a0335dc3b79a100_9806b0fa8f18a18b9b918dba8c42bc67 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411258bcc9d8f8f704f318a0335dc3b79a100_9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:33,801 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411258fcf2b80c8134fc2873c083a1e8cdc3c_9806b0fa8f18a18b9b918dba8c42bc67 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411258fcf2b80c8134fc2873c083a1e8cdc3c_9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:33,802 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125afd305103edf41f785c1110cfd7fda3f_9806b0fa8f18a18b9b918dba8c42bc67 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125afd305103edf41f785c1110cfd7fda3f_9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:33,803 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125c5bbdcbe853d4f799a021104684ce2f4_9806b0fa8f18a18b9b918dba8c42bc67 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125c5bbdcbe853d4f799a021104684ce2f4_9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:33,804 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125de7f2ff4f38749c19c92ec8b4c0e3ba5_9806b0fa8f18a18b9b918dba8c42bc67 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125de7f2ff4f38749c19c92ec8b4c0e3ba5_9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:33,804 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125e196fa7c773a4c16b513b855daf41b32_9806b0fa8f18a18b9b918dba8c42bc67 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125e196fa7c773a4c16b513b855daf41b32_9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:33,805 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125eb0497a3300245c7b8fa2077c37016d5_9806b0fa8f18a18b9b918dba8c42bc67 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125eb0497a3300245c7b8fa2077c37016d5_9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:33,807 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125ee4be1eec10a4442a63fdce76cce6624_9806b0fa8f18a18b9b918dba8c42bc67 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125ee4be1eec10a4442a63fdce76cce6624_9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:33,808 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125f25c3593df774cedb157b5c75b80c83d_9806b0fa8f18a18b9b918dba8c42bc67 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125f25c3593df774cedb157b5c75b80c83d_9806b0fa8f18a18b9b918dba8c42bc67 2024-11-25T12:58:33,808 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-25T12:58:33,810 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=54, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T12:58:33,812 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-25T12:58:33,814 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-25T12:58:33,816 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=54, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T12:58:33,816 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-25T12:58:33,816 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732539513816"}]},"ts":"9223372036854775807"} 2024-11-25T12:58:33,818 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-25T12:58:33,818 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 9806b0fa8f18a18b9b918dba8c42bc67, NAME => 'TestAcidGuarantees,,1732539481102.9806b0fa8f18a18b9b918dba8c42bc67.', STARTKEY => '', ENDKEY => ''}] 2024-11-25T12:58:33,818 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-25T12:58:33,818 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732539513818"}]},"ts":"9223372036854775807"} 2024-11-25T12:58:33,820 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-25T12:58:33,821 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=54, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T12:58:33,822 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 80 msec 2024-11-25T12:58:33,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-25T12:58:33,850 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-25T12:58:33,851 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:58:33,862 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=238 (was 237) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2bb80d27-3d57-c947-53d6-c4472333a06b/cluster_8b8d8747-9aba-2792-61f4-67cb1dbecf7b/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2bb80d27-3d57-c947-53d6-c4472333a06b/cluster_8b8d8747-9aba-2792-61f4-67cb1dbecf7b/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-348484797_22 at /127.0.0.1:51664 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=455 (was 453) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=880 (was 909), ProcessCount=11 (was 11), AvailableMemoryMB=1252 (was 1451) 2024-11-25T12:58:33,871 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=238, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=880, ProcessCount=11, AvailableMemoryMB=1252 2024-11-25T12:58:33,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-25T12:58:33,873 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T12:58:33,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] procedure2.ProcedureExecutor(1139): Stored pid=55, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-25T12:58:33,875 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T12:58:33,875 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:33,875 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 55 2024-11-25T12:58:33,876 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T12:58:33,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-11-25T12:58:33,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742045_1221 (size=959) 2024-11-25T12:58:33,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-11-25T12:58:34,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-11-25T12:58:34,284 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a 2024-11-25T12:58:34,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742046_1222 (size=53) 2024-11-25T12:58:34,304 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:58:34,304 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 21ea60aa3027cec780fb8ce3ec8872d3, disabling compactions & flushes 2024-11-25T12:58:34,304 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:34,304 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:34,304 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. after waiting 0 ms 2024-11-25T12:58:34,304 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:34,304 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:34,304 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 21ea60aa3027cec780fb8ce3ec8872d3: Waiting for close lock at 1732539514304Disabling compacts and flushes for region at 1732539514304Disabling writes for close at 1732539514304Writing region close event to WAL at 1732539514304Closed at 1732539514304 2024-11-25T12:58:34,306 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T12:58:34,306 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732539514306"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732539514306"}]},"ts":"1732539514306"} 2024-11-25T12:58:34,307 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T12:58:34,308 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T12:58:34,308 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732539514308"}]},"ts":"1732539514308"} 2024-11-25T12:58:34,309 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-25T12:58:34,310 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=21ea60aa3027cec780fb8ce3ec8872d3, ASSIGN}] 2024-11-25T12:58:34,311 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=21ea60aa3027cec780fb8ce3ec8872d3, ASSIGN 2024-11-25T12:58:34,311 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=21ea60aa3027cec780fb8ce3ec8872d3, ASSIGN; state=OFFLINE, location=1ff8009e463e,35905,1732539449246; forceNewPlan=false, retain=false 2024-11-25T12:58:34,462 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=21ea60aa3027cec780fb8ce3ec8872d3, regionState=OPENING, regionLocation=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:34,464 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=21ea60aa3027cec780fb8ce3ec8872d3, ASSIGN because future has completed 2024-11-25T12:58:34,464 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; OpenRegionProcedure 21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246}] 2024-11-25T12:58:34,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-11-25T12:58:34,622 INFO [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:34,622 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(7752): Opening region: {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} 2024-11-25T12:58:34,623 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 21ea60aa3027cec780fb8ce3ec8872d3 2024-11-25T12:58:34,623 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:58:34,623 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(7794): checking encryption for 21ea60aa3027cec780fb8ce3ec8872d3 2024-11-25T12:58:34,623 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(7797): checking classloading for 21ea60aa3027cec780fb8ce3ec8872d3 2024-11-25T12:58:34,627 INFO [StoreOpener-21ea60aa3027cec780fb8ce3ec8872d3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 21ea60aa3027cec780fb8ce3ec8872d3 2024-11-25T12:58:34,630 INFO [StoreOpener-21ea60aa3027cec780fb8ce3ec8872d3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 21ea60aa3027cec780fb8ce3ec8872d3 columnFamilyName A 2024-11-25T12:58:34,630 DEBUG [StoreOpener-21ea60aa3027cec780fb8ce3ec8872d3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:34,631 INFO [StoreOpener-21ea60aa3027cec780fb8ce3ec8872d3-1 {}] regionserver.HStore(327): Store=21ea60aa3027cec780fb8ce3ec8872d3/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:58:34,631 INFO [StoreOpener-21ea60aa3027cec780fb8ce3ec8872d3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 21ea60aa3027cec780fb8ce3ec8872d3 2024-11-25T12:58:34,633 INFO [StoreOpener-21ea60aa3027cec780fb8ce3ec8872d3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 21ea60aa3027cec780fb8ce3ec8872d3 columnFamilyName B 2024-11-25T12:58:34,633 DEBUG [StoreOpener-21ea60aa3027cec780fb8ce3ec8872d3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:34,634 INFO [StoreOpener-21ea60aa3027cec780fb8ce3ec8872d3-1 {}] regionserver.HStore(327): Store=21ea60aa3027cec780fb8ce3ec8872d3/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:58:34,634 INFO [StoreOpener-21ea60aa3027cec780fb8ce3ec8872d3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 21ea60aa3027cec780fb8ce3ec8872d3 2024-11-25T12:58:34,636 INFO [StoreOpener-21ea60aa3027cec780fb8ce3ec8872d3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 21ea60aa3027cec780fb8ce3ec8872d3 columnFamilyName C 2024-11-25T12:58:34,636 DEBUG [StoreOpener-21ea60aa3027cec780fb8ce3ec8872d3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:58:34,637 INFO [StoreOpener-21ea60aa3027cec780fb8ce3ec8872d3-1 {}] regionserver.HStore(327): Store=21ea60aa3027cec780fb8ce3ec8872d3/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:58:34,637 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1038): replaying wal for 21ea60aa3027cec780fb8ce3ec8872d3 2024-11-25T12:58:34,638 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3 2024-11-25T12:58:34,639 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3 2024-11-25T12:58:34,639 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1048): stopping wal replay for 21ea60aa3027cec780fb8ce3ec8872d3 2024-11-25T12:58:34,639 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1060): Cleaning up temporary data for 21ea60aa3027cec780fb8ce3ec8872d3 2024-11-25T12:58:34,640 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-25T12:58:34,642 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1093): writing seq id for 21ea60aa3027cec780fb8ce3ec8872d3 2024-11-25T12:58:34,643 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T12:58:34,646 INFO [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1114): Opened 21ea60aa3027cec780fb8ce3ec8872d3; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64073086, jitterRate=-0.04523661732673645}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-25T12:58:34,646 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 21ea60aa3027cec780fb8ce3ec8872d3 2024-11-25T12:58:34,647 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1006): Region open journal for 21ea60aa3027cec780fb8ce3ec8872d3: Running coprocessor pre-open hook at 1732539514623Writing region info on filesystem at 1732539514623Initializing all the Stores at 1732539514626 (+3 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732539514626Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732539514627 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732539514627Cleaning up temporary data from old regions at 1732539514639 (+12 ms)Running coprocessor post-open hooks at 1732539514646 (+7 ms)Region opened successfully at 1732539514646 2024-11-25T12:58:34,648 INFO [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., pid=57, masterSystemTime=1732539514615 2024-11-25T12:58:34,653 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:34,653 INFO [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:34,653 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=21ea60aa3027cec780fb8ce3ec8872d3, regionState=OPEN, openSeqNum=2, regionLocation=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:34,656 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, ppid=56, state=RUNNABLE, hasLock=false; OpenRegionProcedure 21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 because future has completed 2024-11-25T12:58:34,659 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-25T12:58:34,659 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; OpenRegionProcedure 21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 in 193 msec 2024-11-25T12:58:34,661 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=55 2024-11-25T12:58:34,661 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=55, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=21ea60aa3027cec780fb8ce3ec8872d3, ASSIGN in 349 msec 2024-11-25T12:58:34,662 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T12:58:34,662 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732539514662"}]},"ts":"1732539514662"} 2024-11-25T12:58:34,663 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-25T12:58:34,665 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T12:58:34,666 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 792 msec 2024-11-25T12:58:35,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-11-25T12:58:35,011 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:58:35,011 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21c1be0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:58:35,012 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ff8009e463e,46289,-1 for getting cluster id 2024-11-25T12:58:35,012 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:58:35,013 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb972fd5-cdcb-422b-acfe-41fc80d301d9' 2024-11-25T12:58:35,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:58:35,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb972fd5-cdcb-422b-acfe-41fc80d301d9" 2024-11-25T12:58:35,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43de49d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:58:35,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ff8009e463e,46289,-1] 2024-11-25T12:58:35,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:58:35,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:58:35,014 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57390, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:58:35,015 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1129a3f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:58:35,015 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:58:35,016 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,35905,1732539449246, seqNum=-1] 2024-11-25T12:58:35,016 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:58:35,017 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52932, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:58:35,017 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-25T12:58:35,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 1ff8009e463e,46289,1732539448369 2024-11-25T12:58:35,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7e671617 2024-11-25T12:58:35,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-25T12:58:35,019 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57404, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-25T12:58:35,020 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1240be3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:58:35,020 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ff8009e463e,46289,-1 for getting cluster id 2024-11-25T12:58:35,020 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:58:35,021 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb972fd5-cdcb-422b-acfe-41fc80d301d9' 2024-11-25T12:58:35,021 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:58:35,021 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb972fd5-cdcb-422b-acfe-41fc80d301d9" 2024-11-25T12:58:35,021 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71f911b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:58:35,021 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ff8009e463e,46289,-1] 2024-11-25T12:58:35,021 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:58:35,022 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:58:35,022 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57416, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:58:35,023 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44ab8759, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:58:35,024 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2885eb44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:58:35,024 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ff8009e463e,46289,-1 for getting cluster id 2024-11-25T12:58:35,024 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:58:35,024 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb972fd5-cdcb-422b-acfe-41fc80d301d9' 2024-11-25T12:58:35,025 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:58:35,025 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb972fd5-cdcb-422b-acfe-41fc80d301d9" 2024-11-25T12:58:35,025 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51678a7e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:58:35,025 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ff8009e463e,46289,-1] 2024-11-25T12:58:35,025 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:58:35,025 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:58:35,026 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57434, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:58:35,026 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fe9d649, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:58:35,027 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@692a6227, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:58:35,027 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ff8009e463e,46289,-1 for getting cluster id 2024-11-25T12:58:35,027 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:58:35,028 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb972fd5-cdcb-422b-acfe-41fc80d301d9' 2024-11-25T12:58:35,028 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:58:35,028 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb972fd5-cdcb-422b-acfe-41fc80d301d9" 2024-11-25T12:58:35,028 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20300f7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:58:35,028 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ff8009e463e,46289,-1] 2024-11-25T12:58:35,028 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:58:35,028 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:58:35,029 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57446, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:58:35,029 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41af9a5a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:58:35,031 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@128d97bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:58:35,031 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ff8009e463e,46289,-1 for getting cluster id 2024-11-25T12:58:35,031 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:58:35,032 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb972fd5-cdcb-422b-acfe-41fc80d301d9' 2024-11-25T12:58:35,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:58:35,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb972fd5-cdcb-422b-acfe-41fc80d301d9" 2024-11-25T12:58:35,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c94351d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:58:35,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ff8009e463e,46289,-1] 2024-11-25T12:58:35,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:58:35,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:58:35,034 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57470, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:58:35,034 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@526f59dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:58:35,035 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7acbee0d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:58:35,035 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ff8009e463e,46289,-1 for getting cluster id 2024-11-25T12:58:35,036 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:58:35,037 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb972fd5-cdcb-422b-acfe-41fc80d301d9' 2024-11-25T12:58:35,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:58:35,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb972fd5-cdcb-422b-acfe-41fc80d301d9" 2024-11-25T12:58:35,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7de2652e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:58:35,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ff8009e463e,46289,-1] 2024-11-25T12:58:35,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:58:35,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:58:35,038 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57486, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:58:35,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44a2638b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:58:35,039 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47a5c877, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:58:35,039 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ff8009e463e,46289,-1 for getting cluster id 2024-11-25T12:58:35,039 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:58:35,040 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb972fd5-cdcb-422b-acfe-41fc80d301d9' 2024-11-25T12:58:35,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:58:35,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb972fd5-cdcb-422b-acfe-41fc80d301d9" 2024-11-25T12:58:35,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79120bc6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:58:35,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ff8009e463e,46289,-1] 2024-11-25T12:58:35,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:58:35,041 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:58:35,041 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57516, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:58:35,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51240087, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:58:35,043 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@679f7426, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:58:35,043 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ff8009e463e,46289,-1 for getting cluster id 2024-11-25T12:58:35,043 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:58:35,044 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb972fd5-cdcb-422b-acfe-41fc80d301d9' 2024-11-25T12:58:35,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:58:35,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb972fd5-cdcb-422b-acfe-41fc80d301d9" 2024-11-25T12:58:35,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@169a4797, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:58:35,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ff8009e463e,46289,-1] 2024-11-25T12:58:35,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:58:35,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:58:35,045 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57534, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:58:35,046 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@106fb4f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:58:35,046 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b71af22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:58:35,046 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ff8009e463e,46289,-1 for getting cluster id 2024-11-25T12:58:35,047 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:58:35,047 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb972fd5-cdcb-422b-acfe-41fc80d301d9' 2024-11-25T12:58:35,047 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:58:35,047 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb972fd5-cdcb-422b-acfe-41fc80d301d9" 2024-11-25T12:58:35,048 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ce6a633, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:58:35,048 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ff8009e463e,46289,-1] 2024-11-25T12:58:35,048 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:58:35,048 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:58:35,049 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57558, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:58:35,049 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53c27504, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:58:35,051 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@728d338, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:58:35,051 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ff8009e463e,46289,-1 for getting cluster id 2024-11-25T12:58:35,051 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:58:35,052 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb972fd5-cdcb-422b-acfe-41fc80d301d9' 2024-11-25T12:58:35,052 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:58:35,052 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb972fd5-cdcb-422b-acfe-41fc80d301d9" 2024-11-25T12:58:35,053 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42ad7320, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:58:35,053 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ff8009e463e,46289,-1] 2024-11-25T12:58:35,053 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:58:35,053 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:58:35,053 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57588, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:58:35,054 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@645f9e48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:58:35,054 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cc00bc6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:58:35,054 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ff8009e463e,46289,-1 for getting cluster id 2024-11-25T12:58:35,055 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:58:35,055 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb972fd5-cdcb-422b-acfe-41fc80d301d9' 2024-11-25T12:58:35,055 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:58:35,055 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb972fd5-cdcb-422b-acfe-41fc80d301d9" 2024-11-25T12:58:35,055 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b2cf695, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:58:35,055 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ff8009e463e,46289,-1] 2024-11-25T12:58:35,056 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:58:35,056 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:58:35,056 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57612, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:58:35,057 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1896c49a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:58:35,060 DEBUG [Thread-1050 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:58:35,060 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,35905,1732539449246, seqNum=-1] 2024-11-25T12:58:35,061 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:58:35,062 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52942, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:58:35,062 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:58:35,062 DEBUG [Thread-1059 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:58:35,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,35905,1732539449246, seqNum=-1] 2024-11-25T12:58:35,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:58:35,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-11-25T12:58:35,064 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2] 2024-11-25T12:58:35,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-25T12:58:35,065 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52950, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:58:35,065 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:58:35,066 DEBUG [Thread-1063 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:58:35,066 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:58:35,067 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:58:35,067 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,35905,1732539449246, seqNum=-1] 2024-11-25T12:58:35,067 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2] 2024-11-25T12:58:35,067 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:58:35,068 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52952, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:58:35,068 DEBUG [Thread-1065 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:58:35,069 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,35905,1732539449246, seqNum=-1] 2024-11-25T12:58:35,069 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2] 2024-11-25T12:58:35,069 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:58:35,070 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52968, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:58:35,072 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2] 2024-11-25T12:58:35,076 DEBUG [Thread-1054 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:58:35,076 DEBUG [Thread-1067 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:58:35,077 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,35905,1732539449246, seqNum=-1] 2024-11-25T12:58:35,077 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:58:35,077 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,35905,1732539449246, seqNum=-1] 2024-11-25T12:58:35,078 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:58:35,078 DEBUG [Thread-1052 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:58:35,079 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52996, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:58:35,079 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52984, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:58:35,079 DEBUG [Thread-1056 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:58:35,079 DEBUG [Thread-1048 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:58:35,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,35905,1732539449246, seqNum=-1] 2024-11-25T12:58:35,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:58:35,081 DEBUG [Thread-1052 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,35905,1732539449246, seqNum=-1] 2024-11-25T12:58:35,081 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2] 2024-11-25T12:58:35,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2] 2024-11-25T12:58:35,081 DEBUG [Thread-1052 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:58:35,081 DEBUG [Thread-1061 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:58:35,082 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,35905,1732539449246, seqNum=-1] 2024-11-25T12:58:35,082 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:58:35,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,35905,1732539449246, seqNum=-1] 2024-11-25T12:58:35,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:58:35,084 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53022, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:58:35,085 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53014, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:58:35,085 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53004, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:58:35,086 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53034, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:58:35,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2] 2024-11-25T12:58:35,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2] 2024-11-25T12:58:35,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2] 2024-11-25T12:58:35,088 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2] 2024-11-25T12:58:35,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 21ea60aa3027cec780fb8ce3ec8872d3 2024-11-25T12:58:35,092 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:58:35,092 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 21ea60aa3027cec780fb8ce3ec8872d3 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-25T12:58:35,130 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/48b4926c83854b2690630b6c6b249609 is 50, key is test_row_0/A:col10/1732539515091/Put/seqid=0 2024-11-25T12:58:35,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:35,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53014 deadline: 1732539575150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:35,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:35,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:35,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:35,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:35,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53022 deadline: 1732539575158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:35,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:35,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742047_1223 (size=9657) 2024-11-25T12:58:35,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:35,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:35,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:35,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52942 deadline: 1732539575159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:35,169 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:35,169 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:35,169 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:35,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-25T12:58:35,171 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/48b4926c83854b2690630b6c6b249609 2024-11-25T12:58:35,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:35,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1732539575161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:35,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:35,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:35,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:35,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:35,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52996 deadline: 1732539575166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:35,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:35,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:35,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:35,221 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/eb655ad7125f412aae88a1f988b3b88c is 50, key is test_row_0/B:col10/1732539515091/Put/seqid=0 2024-11-25T12:58:35,222 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-25T12:58:35,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:35,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. as already flushing 2024-11-25T12:58:35,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:35,222 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:35,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:35,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:35,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742048_1224 (size=9657) 2024-11-25T12:58:35,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:35,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53014 deadline: 1732539575270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:35,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:35,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:35,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:35,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:35,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53022 deadline: 1732539575282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:35,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:35,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52942 deadline: 1732539575282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:35,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:35,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:35,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:35,285 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:35,285 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:35,285 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:35,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:35,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1732539575282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:35,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:35,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52996 deadline: 1732539575282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:35,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:35,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:35,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:35,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:35,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:35,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:35,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-25T12:58:35,388 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-25T12:58:35,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:35,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. as already flushing 2024-11-25T12:58:35,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:35,388 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:35,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:35,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:35,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:35,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53014 deadline: 1732539575480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:35,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:35,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:35,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:35,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:35,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53022 deadline: 1732539575491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:35,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:35,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:35,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:35,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:35,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52942 deadline: 1732539575491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:35,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:35,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1732539575491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:35,493 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:35,493 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:35,493 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:35,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:35,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:35,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:35,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:35,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52996 deadline: 1732539575491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:35,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:35,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:35,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:35,545 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-25T12:58:35,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:35,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. as already flushing 2024-11-25T12:58:35,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:35,546 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:35,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:35,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:35,657 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/eb655ad7125f412aae88a1f988b3b88c 2024-11-25T12:58:35,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-25T12:58:35,700 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-25T12:58:35,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:35,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. as already flushing 2024-11-25T12:58:35,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:35,701 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:35,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:35,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:35,710 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/7275d66c4c0b4ba397a86c10ddbb6a0b is 50, key is test_row_0/C:col10/1732539515091/Put/seqid=0 2024-11-25T12:58:35,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742049_1225 (size=9657) 2024-11-25T12:58:35,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:35,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53014 deadline: 1732539575791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:35,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:35,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:35,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:35,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:35,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53022 deadline: 1732539575802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:35,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:35,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:35,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:35,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:35,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:35,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1732539575802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:35,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:35,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:35,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:35,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:35,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52996 deadline: 1732539575802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:35,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52942 deadline: 1732539575802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:35,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:35,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:35,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:35,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:35,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:35,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:35,875 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-25T12:58:35,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:35,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. as already flushing 2024-11-25T12:58:35,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:35,875 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:35,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:35,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:36,042 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-25T12:58:36,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:36,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. as already flushing 2024-11-25T12:58:36,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:36,043 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:36,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:36,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:36,168 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/7275d66c4c0b4ba397a86c10ddbb6a0b 2024-11-25T12:58:36,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/48b4926c83854b2690630b6c6b249609 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/48b4926c83854b2690630b6c6b249609 2024-11-25T12:58:36,178 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/48b4926c83854b2690630b6c6b249609, entries=100, sequenceid=13, filesize=9.4 K 2024-11-25T12:58:36,178 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-25T12:58:36,179 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/eb655ad7125f412aae88a1f988b3b88c as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/eb655ad7125f412aae88a1f988b3b88c 2024-11-25T12:58:36,183 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/eb655ad7125f412aae88a1f988b3b88c, entries=100, sequenceid=13, filesize=9.4 K 2024-11-25T12:58:36,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/7275d66c4c0b4ba397a86c10ddbb6a0b as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/7275d66c4c0b4ba397a86c10ddbb6a0b 2024-11-25T12:58:36,190 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/7275d66c4c0b4ba397a86c10ddbb6a0b, entries=100, sequenceid=13, filesize=9.4 K 2024-11-25T12:58:36,191 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 21ea60aa3027cec780fb8ce3ec8872d3 in 1099ms, sequenceid=13, compaction requested=false 2024-11-25T12:58:36,191 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 21ea60aa3027cec780fb8ce3ec8872d3: 2024-11-25T12:58:36,196 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-25T12:58:36,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:36,196 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2902): Flushing 21ea60aa3027cec780fb8ce3ec8872d3 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-25T12:58:36,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-25T12:58:36,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/64e408fb3daf47a89800635736b8f450 is 50, key is test_row_0/A:col10/1732539515144/Put/seqid=0 2024-11-25T12:58:36,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742050_1226 (size=12001) 2024-11-25T12:58:36,246 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/64e408fb3daf47a89800635736b8f450 2024-11-25T12:58:36,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/70c0dc94245346a6976795c720879ce6 is 50, key is test_row_0/B:col10/1732539515144/Put/seqid=0 2024-11-25T12:58:36,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742051_1227 (size=12001) 2024-11-25T12:58:36,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 21ea60aa3027cec780fb8ce3ec8872d3 2024-11-25T12:58:36,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. as already flushing 2024-11-25T12:58:36,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:36,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53014 deadline: 1732539576323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:36,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:36,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:36,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:36,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:36,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53022 deadline: 1732539576325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:36,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:36,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1732539576325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:36,330 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:36,330 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:36,330 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:36,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:36,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:36,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:36,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:36,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52996 deadline: 1732539576326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:36,331 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:36,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:36,331 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:36,331 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:36,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52942 deadline: 1732539576326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:36,331 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:36,331 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:36,331 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:36,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:36,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53014 deadline: 1732539576440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:36,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:36,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:36,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:36,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:36,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53022 deadline: 1732539576440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:36,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:36,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:36,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:36,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:36,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52996 deadline: 1732539576440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:36,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:36,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:36,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:36,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:36,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1732539576440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:36,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:36,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52942 deadline: 1732539576440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:36,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:36,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:36,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:36,445 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:36,445 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:36,445 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:36,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:36,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53014 deadline: 1732539576655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:36,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:36,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:36,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:36,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:36,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52942 deadline: 1732539576656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:36,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:36,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1732539576656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:36,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:36,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53022 deadline: 1732539576656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:36,660 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:36,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:36,660 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:36,660 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:36,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52996 deadline: 1732539576656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:36,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:36,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:36,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:36,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:36,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:36,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:36,661 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:36,661 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:36,661 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:36,683 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/70c0dc94245346a6976795c720879ce6 2024-11-25T12:58:36,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/9d2b1a3435404800ad54f6bbe784b524 is 50, key is test_row_0/C:col10/1732539515144/Put/seqid=0 2024-11-25T12:58:36,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742052_1228 (size=12001) 2024-11-25T12:58:36,741 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/9d2b1a3435404800ad54f6bbe784b524 2024-11-25T12:58:36,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/64e408fb3daf47a89800635736b8f450 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/64e408fb3daf47a89800635736b8f450 2024-11-25T12:58:36,773 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/64e408fb3daf47a89800635736b8f450, entries=150, sequenceid=37, filesize=11.7 K 2024-11-25T12:58:36,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/70c0dc94245346a6976795c720879ce6 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/70c0dc94245346a6976795c720879ce6 2024-11-25T12:58:36,790 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/70c0dc94245346a6976795c720879ce6, entries=150, sequenceid=37, filesize=11.7 K 2024-11-25T12:58:36,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/9d2b1a3435404800ad54f6bbe784b524 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/9d2b1a3435404800ad54f6bbe784b524 2024-11-25T12:58:36,821 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/9d2b1a3435404800ad54f6bbe784b524, entries=150, sequenceid=37, filesize=11.7 K 2024-11-25T12:58:36,830 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 21ea60aa3027cec780fb8ce3ec8872d3 in 634ms, sequenceid=37, compaction requested=false 2024-11-25T12:58:36,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for 21ea60aa3027cec780fb8ce3ec8872d3: 2024-11-25T12:58:36,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:36,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-25T12:58:36,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-25T12:58:36,848 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-25T12:58:36,848 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7690 sec 2024-11-25T12:58:36,859 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 1.7930 sec 2024-11-25T12:58:36,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 21ea60aa3027cec780fb8ce3ec8872d3 2024-11-25T12:58:36,973 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:58:36,973 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 21ea60aa3027cec780fb8ce3ec8872d3 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-25T12:58:36,993 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/6e413a1d2b09485b897a50aa02c350ad is 50, key is test_row_0/A:col10/1732539516325/Put/seqid=0 2024-11-25T12:58:37,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742053_1229 (size=14341) 2024-11-25T12:58:37,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:37,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1732539577021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:37,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:37,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:37,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53014 deadline: 1732539577023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:37,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:37,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:37,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:37,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:37,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:37,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:37,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:37,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52996 deadline: 1732539577052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:37,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:37,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52942 deadline: 1732539577053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:37,067 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:37,067 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:37,067 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:37,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:37,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:37,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:37,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53022 deadline: 1732539577052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:37,068 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:37,068 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:37,068 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:37,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:37,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53014 deadline: 1732539577150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:37,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:37,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1732539577151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:37,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:37,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:37,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:37,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:37,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:37,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:37,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:37,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52942 deadline: 1732539577169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:37,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:37,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:37,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:37,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:37,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53022 deadline: 1732539577170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:37,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:37,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:37,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52996 deadline: 1732539577171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:37,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:37,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:37,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:37,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:37,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:37,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-25T12:58:37,220 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:58:37,221 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:58:37,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-11-25T12:58:37,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-25T12:58:37,222 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:58:37,223 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:58:37,224 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:58:37,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-25T12:58:37,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:37,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1732539577360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:37,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:37,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:37,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:37,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53014 deadline: 1732539577360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:37,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:37,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:37,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:37,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:37,376 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-25T12:58:37,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:37,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. as already flushing 2024-11-25T12:58:37,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:37,376 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:37,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:37,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:37,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:37,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52942 deadline: 1732539577390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:37,392 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:37,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:37,392 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:37,392 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:37,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52996 deadline: 1732539577390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:37,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:37,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:37,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:37,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:37,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53022 deadline: 1732539577390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:37,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:37,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:37,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:37,402 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/6e413a1d2b09485b897a50aa02c350ad 2024-11-25T12:58:37,417 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/23581281edce4ea482676d8eca8ec78e is 50, key is test_row_0/B:col10/1732539516325/Put/seqid=0 2024-11-25T12:58:37,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742054_1230 (size=12001) 2024-11-25T12:58:37,433 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/23581281edce4ea482676d8eca8ec78e 2024-11-25T12:58:37,441 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/6b9119d676114d96a43ae469a51a6c08 is 50, key is test_row_0/C:col10/1732539516325/Put/seqid=0 2024-11-25T12:58:37,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742055_1231 (size=12001) 2024-11-25T12:58:37,529 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-25T12:58:37,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:37,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. as already flushing 2024-11-25T12:58:37,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:37,529 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:37,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:37,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:37,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-25T12:58:37,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:37,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1732539577670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:37,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:37,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:37,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:37,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:37,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53014 deadline: 1732539577670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:37,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:37,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:37,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:37,683 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-25T12:58:37,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:37,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. as already flushing 2024-11-25T12:58:37,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:37,684 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:37,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:37,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:37,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:37,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:37,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52996 deadline: 1732539577700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:37,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52942 deadline: 1732539577700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:37,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:37,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:37,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:37,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:37,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53022 deadline: 1732539577700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:37,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:37,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:37,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:37,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:37,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:37,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:37,842 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-25T12:58:37,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:37,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. as already flushing 2024-11-25T12:58:37,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:37,843 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:37,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:37,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:37,849 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/6b9119d676114d96a43ae469a51a6c08 2024-11-25T12:58:37,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-25T12:58:37,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/6e413a1d2b09485b897a50aa02c350ad as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/6e413a1d2b09485b897a50aa02c350ad 2024-11-25T12:58:37,883 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/6e413a1d2b09485b897a50aa02c350ad, entries=200, sequenceid=51, filesize=14.0 K 2024-11-25T12:58:37,885 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/23581281edce4ea482676d8eca8ec78e as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/23581281edce4ea482676d8eca8ec78e 2024-11-25T12:58:37,891 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/23581281edce4ea482676d8eca8ec78e, entries=150, sequenceid=51, filesize=11.7 K 2024-11-25T12:58:37,892 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/6b9119d676114d96a43ae469a51a6c08 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/6b9119d676114d96a43ae469a51a6c08 2024-11-25T12:58:37,901 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/6b9119d676114d96a43ae469a51a6c08, entries=150, sequenceid=51, filesize=11.7 K 2024-11-25T12:58:37,902 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 21ea60aa3027cec780fb8ce3ec8872d3 in 929ms, sequenceid=51, compaction requested=true 2024-11-25T12:58:37,902 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 21ea60aa3027cec780fb8ce3ec8872d3: 2024-11-25T12:58:37,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21ea60aa3027cec780fb8ce3ec8872d3:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:58:37,902 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:58:37,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:58:37,902 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:58:37,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21ea60aa3027cec780fb8ce3ec8872d3:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:58:37,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:58:37,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21ea60aa3027cec780fb8ce3ec8872d3:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:58:37,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:58:37,904 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:58:37,904 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35999 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:58:37,904 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1541): 21ea60aa3027cec780fb8ce3ec8872d3/A is initiating minor compaction (all files) 2024-11-25T12:58:37,904 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): 21ea60aa3027cec780fb8ce3ec8872d3/B is initiating minor compaction (all files) 2024-11-25T12:58:37,904 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21ea60aa3027cec780fb8ce3ec8872d3/A in TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:37,904 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/48b4926c83854b2690630b6c6b249609, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/64e408fb3daf47a89800635736b8f450, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/6e413a1d2b09485b897a50aa02c350ad] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp, totalSize=35.2 K 2024-11-25T12:58:37,904 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21ea60aa3027cec780fb8ce3ec8872d3/B in TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:37,904 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/eb655ad7125f412aae88a1f988b3b88c, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/70c0dc94245346a6976795c720879ce6, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/23581281edce4ea482676d8eca8ec78e] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp, totalSize=32.9 K 2024-11-25T12:58:37,905 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting eb655ad7125f412aae88a1f988b3b88c, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732539515077 2024-11-25T12:58:37,905 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48b4926c83854b2690630b6c6b249609, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732539515077 2024-11-25T12:58:37,906 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 70c0dc94245346a6976795c720879ce6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732539515144 2024-11-25T12:58:37,906 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64e408fb3daf47a89800635736b8f450, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732539515144 2024-11-25T12:58:37,908 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 23581281edce4ea482676d8eca8ec78e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732539516316 2024-11-25T12:58:37,908 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e413a1d2b09485b897a50aa02c350ad, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732539516316 2024-11-25T12:58:37,942 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21ea60aa3027cec780fb8ce3ec8872d3#B#compaction#186 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:58:37,942 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/b452326c17ad45dc9b77bbdf2d1a6bbd is 50, key is test_row_0/B:col10/1732539516325/Put/seqid=0 2024-11-25T12:58:37,950 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21ea60aa3027cec780fb8ce3ec8872d3#A#compaction#187 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:58:37,950 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/dabe892d4a434d8ea8a4eafe25b7097a is 50, key is test_row_0/A:col10/1732539516325/Put/seqid=0 2024-11-25T12:58:38,008 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-25T12:58:38,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:38,009 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing 21ea60aa3027cec780fb8ce3ec8872d3 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-25T12:58:38,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742056_1232 (size=12104) 2024-11-25T12:58:38,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742057_1233 (size=12104) 2024-11-25T12:58:38,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/dcb716eb302d44afa8df29008645a28c is 50, key is test_row_0/A:col10/1732539517012/Put/seqid=0 2024-11-25T12:58:38,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742058_1234 (size=12001) 2024-11-25T12:58:38,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 21ea60aa3027cec780fb8ce3ec8872d3 2024-11-25T12:58:38,190 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. as already flushing 2024-11-25T12:58:38,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:38,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53022 deadline: 1732539578281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:38,305 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:38,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:38,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:38,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:38,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1732539578297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:38,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:38,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:38,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:38,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:38,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53014 deadline: 1732539578297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:38,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:38,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52996 deadline: 1732539578306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:38,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:38,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:38,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:38,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:38,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:38,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:38,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:38,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52942 deadline: 1732539578309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:38,325 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:38,325 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:38,325 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:38,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-25T12:58:38,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:38,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53022 deadline: 1732539578410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:38,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:38,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:38,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:38,418 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/b452326c17ad45dc9b77bbdf2d1a6bbd as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/b452326c17ad45dc9b77bbdf2d1a6bbd 2024-11-25T12:58:38,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:38,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1732539578419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:38,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:38,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:38,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:38,428 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21ea60aa3027cec780fb8ce3ec8872d3/B of 21ea60aa3027cec780fb8ce3ec8872d3 into b452326c17ad45dc9b77bbdf2d1a6bbd(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:58:38,428 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21ea60aa3027cec780fb8ce3ec8872d3: 2024-11-25T12:58:38,428 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., storeName=21ea60aa3027cec780fb8ce3ec8872d3/B, priority=13, startTime=1732539517902; duration=0sec 2024-11-25T12:58:38,428 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:58:38,428 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21ea60aa3027cec780fb8ce3ec8872d3:B 2024-11-25T12:58:38,428 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:58:38,433 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:58:38,433 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): 21ea60aa3027cec780fb8ce3ec8872d3/C is initiating minor compaction (all files) 2024-11-25T12:58:38,433 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21ea60aa3027cec780fb8ce3ec8872d3/C in TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:38,433 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/7275d66c4c0b4ba397a86c10ddbb6a0b, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/9d2b1a3435404800ad54f6bbe784b524, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/6b9119d676114d96a43ae469a51a6c08] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp, totalSize=32.9 K 2024-11-25T12:58:38,436 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/dabe892d4a434d8ea8a4eafe25b7097a as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/dabe892d4a434d8ea8a4eafe25b7097a 2024-11-25T12:58:38,439 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 7275d66c4c0b4ba397a86c10ddbb6a0b, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732539515077 2024-11-25T12:58:38,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:38,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52996 deadline: 1732539578430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:38,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:38,440 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:38,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53014 deadline: 1732539578430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:38,440 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:38,440 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:38,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:38,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52942 deadline: 1732539578430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:38,440 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:38,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:38,440 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:38,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:38,440 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:38,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:38,441 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d2b1a3435404800ad54f6bbe784b524, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732539515144 2024-11-25T12:58:38,441 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b9119d676114d96a43ae469a51a6c08, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732539516316 2024-11-25T12:58:38,450 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21ea60aa3027cec780fb8ce3ec8872d3#C#compaction#189 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:58:38,451 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/f020af86f275434a845156e51ca1064b is 50, key is test_row_0/C:col10/1732539516325/Put/seqid=0 2024-11-25T12:58:38,451 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21ea60aa3027cec780fb8ce3ec8872d3/A of 21ea60aa3027cec780fb8ce3ec8872d3 into dabe892d4a434d8ea8a4eafe25b7097a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:58:38,452 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21ea60aa3027cec780fb8ce3ec8872d3: 2024-11-25T12:58:38,452 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., storeName=21ea60aa3027cec780fb8ce3ec8872d3/A, priority=13, startTime=1732539517902; duration=0sec 2024-11-25T12:58:38,452 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:58:38,452 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21ea60aa3027cec780fb8ce3ec8872d3:A 2024-11-25T12:58:38,479 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/dcb716eb302d44afa8df29008645a28c 2024-11-25T12:58:38,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742059_1235 (size=12104) 2024-11-25T12:58:38,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/8fe50a49de2847608b4c4bc2cb0c34e3 is 50, key is test_row_0/B:col10/1732539517012/Put/seqid=0 2024-11-25T12:58:38,507 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/f020af86f275434a845156e51ca1064b as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/f020af86f275434a845156e51ca1064b 2024-11-25T12:58:38,513 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21ea60aa3027cec780fb8ce3ec8872d3/C of 21ea60aa3027cec780fb8ce3ec8872d3 into f020af86f275434a845156e51ca1064b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:58:38,513 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21ea60aa3027cec780fb8ce3ec8872d3: 2024-11-25T12:58:38,513 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., storeName=21ea60aa3027cec780fb8ce3ec8872d3/C, priority=13, startTime=1732539517903; duration=0sec 2024-11-25T12:58:38,513 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:58:38,513 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21ea60aa3027cec780fb8ce3ec8872d3:C 2024-11-25T12:58:38,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742060_1236 (size=12001) 2024-11-25T12:58:38,545 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/8fe50a49de2847608b4c4bc2cb0c34e3 2024-11-25T12:58:38,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/7c2c487f385b473ea216253f051ab9f8 is 50, key is test_row_0/C:col10/1732539517012/Put/seqid=0 2024-11-25T12:58:38,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742061_1237 (size=12001) 2024-11-25T12:58:38,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:38,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53022 deadline: 1732539578620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:38,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:38,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:38,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:38,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:38,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1732539578630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:38,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:38,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:38,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:38,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:38,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52996 deadline: 1732539578650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:38,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:38,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:38,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:38,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:38,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53014 deadline: 1732539578650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:38,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:38,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52942 deadline: 1732539578651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:38,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:38,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:38,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:38,656 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:38,656 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:38,656 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:38,720 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T12:58:38,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:38,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53022 deadline: 1732539578930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:38,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:38,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:38,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:38,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:38,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1732539578942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:38,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:38,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:38,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:38,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:38,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52996 deadline: 1732539578962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:38,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:38,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53014 deadline: 1732539578962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:38,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:38,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52942 deadline: 1732539578962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:38,968 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:38,968 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:38,969 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:38,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:38,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:38,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:38,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:38,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:38,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:39,018 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/7c2c487f385b473ea216253f051ab9f8 2024-11-25T12:58:39,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/dcb716eb302d44afa8df29008645a28c as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/dcb716eb302d44afa8df29008645a28c 2024-11-25T12:58:39,031 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/dcb716eb302d44afa8df29008645a28c, entries=150, sequenceid=73, filesize=11.7 K 2024-11-25T12:58:39,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/8fe50a49de2847608b4c4bc2cb0c34e3 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/8fe50a49de2847608b4c4bc2cb0c34e3 2024-11-25T12:58:39,039 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/8fe50a49de2847608b4c4bc2cb0c34e3, entries=150, sequenceid=73, filesize=11.7 K 2024-11-25T12:58:39,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/7c2c487f385b473ea216253f051ab9f8 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/7c2c487f385b473ea216253f051ab9f8 2024-11-25T12:58:39,051 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/7c2c487f385b473ea216253f051ab9f8, entries=150, sequenceid=73, filesize=11.7 K 2024-11-25T12:58:39,052 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=87.22 KB/89310 for 21ea60aa3027cec780fb8ce3ec8872d3 in 1043ms, sequenceid=73, compaction requested=false 2024-11-25T12:58:39,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for 21ea60aa3027cec780fb8ce3ec8872d3: 2024-11-25T12:58:39,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:39,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-11-25T12:58:39,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-11-25T12:58:39,056 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-25T12:58:39,056 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8300 sec 2024-11-25T12:58:39,058 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 1.8350 sec 2024-11-25T12:58:39,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-25T12:58:39,384 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:58:39,384 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:58:39,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-11-25T12:58:39,387 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:58:39,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-25T12:58:39,388 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:58:39,388 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:58:39,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 21ea60aa3027cec780fb8ce3ec8872d3 2024-11-25T12:58:39,464 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:58:39,464 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 21ea60aa3027cec780fb8ce3ec8872d3 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-25T12:58:39,477 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/0b138a8aa27d4da6823bed5e6930c09b is 50, key is test_row_0/A:col10/1732539518198/Put/seqid=0 2024-11-25T12:58:39,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742062_1238 (size=14341) 2024-11-25T12:58:39,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-25T12:58:39,492 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/0b138a8aa27d4da6823bed5e6930c09b 2024-11-25T12:58:39,504 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/7fab9161fe6443768b91bcafd461d957 is 50, key is test_row_0/B:col10/1732539518198/Put/seqid=0 2024-11-25T12:58:39,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:39,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53014 deadline: 1732539579520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:39,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:39,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:39,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53022 deadline: 1732539579521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:39,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52942 deadline: 1732539579511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:39,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:39,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1732539579521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:39,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:39,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:39,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:39,527 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:39,527 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:39,527 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:39,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:39,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:39,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:39,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:39,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52996 deadline: 1732539579521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:39,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:39,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:39,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:39,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:39,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:39,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:39,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742063_1239 (size=12001) 2024-11-25T12:58:39,542 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/7fab9161fe6443768b91bcafd461d957 2024-11-25T12:58:39,545 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-25T12:58:39,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:39,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. as already flushing 2024-11-25T12:58:39,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:39,551 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:39,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:39,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:39,569 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/3c5035de040d42dba55c5d00667bc2ec is 50, key is test_row_0/C:col10/1732539518198/Put/seqid=0 2024-11-25T12:58:39,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742064_1240 (size=12001) 2024-11-25T12:58:39,593 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/3c5035de040d42dba55c5d00667bc2ec 2024-11-25T12:58:39,599 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/0b138a8aa27d4da6823bed5e6930c09b as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/0b138a8aa27d4da6823bed5e6930c09b 2024-11-25T12:58:39,615 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/0b138a8aa27d4da6823bed5e6930c09b, entries=200, sequenceid=93, filesize=14.0 K 2024-11-25T12:58:39,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:39,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53022 deadline: 1732539579630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:39,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/7fab9161fe6443768b91bcafd461d957 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/7fab9161fe6443768b91bcafd461d957 2024-11-25T12:58:39,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:39,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:39,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:39,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:39,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52942 deadline: 1732539579630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:39,643 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:39,643 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:39,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:39,643 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:39,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1732539579631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:39,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:39,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52996 deadline: 1732539579645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:39,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:39,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53014 deadline: 1732539579645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:39,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:39,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:39,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:39,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:39,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:39,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:39,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:39,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:39,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:39,672 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/7fab9161fe6443768b91bcafd461d957, entries=150, sequenceid=93, filesize=11.7 K 2024-11-25T12:58:39,677 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/3c5035de040d42dba55c5d00667bc2ec as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/3c5035de040d42dba55c5d00667bc2ec 2024-11-25T12:58:39,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-25T12:58:39,706 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/3c5035de040d42dba55c5d00667bc2ec, entries=150, sequenceid=93, filesize=11.7 K 2024-11-25T12:58:39,706 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-25T12:58:39,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:39,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. as already flushing 2024-11-25T12:58:39,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:39,707 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:39,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:39,708 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 21ea60aa3027cec780fb8ce3ec8872d3 in 244ms, sequenceid=93, compaction requested=true 2024-11-25T12:58:39,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:39,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 21ea60aa3027cec780fb8ce3ec8872d3: 2024-11-25T12:58:39,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21ea60aa3027cec780fb8ce3ec8872d3:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:58:39,708 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:58:39,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:58:39,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21ea60aa3027cec780fb8ce3ec8872d3:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:58:39,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:58:39,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21ea60aa3027cec780fb8ce3ec8872d3:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:58:39,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-25T12:58:39,708 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:58:39,710 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:58:39,710 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1541): 21ea60aa3027cec780fb8ce3ec8872d3/A is initiating minor compaction (all files) 2024-11-25T12:58:39,710 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21ea60aa3027cec780fb8ce3ec8872d3/A in TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:39,710 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/dabe892d4a434d8ea8a4eafe25b7097a, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/dcb716eb302d44afa8df29008645a28c, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/0b138a8aa27d4da6823bed5e6930c09b] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp, totalSize=37.5 K 2024-11-25T12:58:39,711 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:58:39,711 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting dabe892d4a434d8ea8a4eafe25b7097a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732539516316 2024-11-25T12:58:39,711 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): 21ea60aa3027cec780fb8ce3ec8872d3/B is initiating minor compaction (all files) 2024-11-25T12:58:39,711 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21ea60aa3027cec780fb8ce3ec8872d3/B in TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:39,711 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/b452326c17ad45dc9b77bbdf2d1a6bbd, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/8fe50a49de2847608b4c4bc2cb0c34e3, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/7fab9161fe6443768b91bcafd461d957] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp, totalSize=35.3 K 2024-11-25T12:58:39,711 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting dcb716eb302d44afa8df29008645a28c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1732539517012 2024-11-25T12:58:39,711 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b138a8aa27d4da6823bed5e6930c09b, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732539518198 2024-11-25T12:58:39,711 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting b452326c17ad45dc9b77bbdf2d1a6bbd, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732539516316 2024-11-25T12:58:39,712 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 8fe50a49de2847608b4c4bc2cb0c34e3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1732539517012 2024-11-25T12:58:39,713 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 7fab9161fe6443768b91bcafd461d957, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732539518198 2024-11-25T12:58:39,725 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21ea60aa3027cec780fb8ce3ec8872d3#A#compaction#195 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:58:39,726 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/34340069d0e043d4ac4ab64128a808f8 is 50, key is test_row_0/A:col10/1732539518198/Put/seqid=0 2024-11-25T12:58:39,733 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21ea60aa3027cec780fb8ce3ec8872d3#B#compaction#196 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:58:39,734 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/44e1280f7b5c47988c86898d9492e6b1 is 50, key is test_row_0/B:col10/1732539518198/Put/seqid=0 2024-11-25T12:58:39,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742065_1241 (size=12207) 2024-11-25T12:58:39,785 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/34340069d0e043d4ac4ab64128a808f8 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/34340069d0e043d4ac4ab64128a808f8 2024-11-25T12:58:39,797 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21ea60aa3027cec780fb8ce3ec8872d3/A of 21ea60aa3027cec780fb8ce3ec8872d3 into 34340069d0e043d4ac4ab64128a808f8(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:58:39,797 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21ea60aa3027cec780fb8ce3ec8872d3: 2024-11-25T12:58:39,797 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., storeName=21ea60aa3027cec780fb8ce3ec8872d3/A, priority=13, startTime=1732539519708; duration=0sec 2024-11-25T12:58:39,797 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:58:39,797 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21ea60aa3027cec780fb8ce3ec8872d3:A 2024-11-25T12:58:39,798 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:58:39,799 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:58:39,799 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1541): 21ea60aa3027cec780fb8ce3ec8872d3/C is initiating minor compaction (all files) 2024-11-25T12:58:39,799 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21ea60aa3027cec780fb8ce3ec8872d3/C in TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:39,800 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/f020af86f275434a845156e51ca1064b, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/7c2c487f385b473ea216253f051ab9f8, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/3c5035de040d42dba55c5d00667bc2ec] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp, totalSize=35.3 K 2024-11-25T12:58:39,800 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting f020af86f275434a845156e51ca1064b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732539516316 2024-11-25T12:58:39,800 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c2c487f385b473ea216253f051ab9f8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1732539517012 2024-11-25T12:58:39,801 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c5035de040d42dba55c5d00667bc2ec, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732539518198 2024-11-25T12:58:39,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742066_1242 (size=12207) 2024-11-25T12:58:39,812 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/44e1280f7b5c47988c86898d9492e6b1 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/44e1280f7b5c47988c86898d9492e6b1 2024-11-25T12:58:39,818 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21ea60aa3027cec780fb8ce3ec8872d3/B of 21ea60aa3027cec780fb8ce3ec8872d3 into 44e1280f7b5c47988c86898d9492e6b1(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:58:39,818 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21ea60aa3027cec780fb8ce3ec8872d3: 2024-11-25T12:58:39,818 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., storeName=21ea60aa3027cec780fb8ce3ec8872d3/B, priority=13, startTime=1732539519708; duration=0sec 2024-11-25T12:58:39,818 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:58:39,819 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21ea60aa3027cec780fb8ce3ec8872d3:B 2024-11-25T12:58:39,834 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21ea60aa3027cec780fb8ce3ec8872d3#C#compaction#197 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:58:39,838 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/2ffb2f8c2f044c2581207e18c41902d5 is 50, key is test_row_0/C:col10/1732539518198/Put/seqid=0 2024-11-25T12:58:39,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 21ea60aa3027cec780fb8ce3ec8872d3 2024-11-25T12:58:39,844 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:58:39,844 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 21ea60aa3027cec780fb8ce3ec8872d3 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-25T12:58:39,861 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/75b76a041c674850b02efce9543325ce is 50, key is test_row_0/A:col10/1732539519519/Put/seqid=0 2024-11-25T12:58:39,861 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-25T12:58:39,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:39,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. as already flushing 2024-11-25T12:58:39,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:39,862 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:39,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:39,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:39,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:39,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53022 deadline: 1732539579874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:39,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:39,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52942 deadline: 1732539579872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:39,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:39,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1732539579875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:39,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:39,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:39,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:39,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:39,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:39,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:39,882 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:39,882 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:39,882 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:39,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:39,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52996 deadline: 1732539579877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:39,883 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:39,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:39,883 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:39,883 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:39,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53014 deadline: 1732539579878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:39,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:39,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:39,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:39,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742067_1243 (size=12207) 2024-11-25T12:58:39,899 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/2ffb2f8c2f044c2581207e18c41902d5 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/2ffb2f8c2f044c2581207e18c41902d5 2024-11-25T12:58:39,906 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21ea60aa3027cec780fb8ce3ec8872d3/C of 21ea60aa3027cec780fb8ce3ec8872d3 into 2ffb2f8c2f044c2581207e18c41902d5(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:58:39,906 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21ea60aa3027cec780fb8ce3ec8872d3: 2024-11-25T12:58:39,907 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., storeName=21ea60aa3027cec780fb8ce3ec8872d3/C, priority=13, startTime=1732539519708; duration=0sec 2024-11-25T12:58:39,907 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:58:39,907 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21ea60aa3027cec780fb8ce3ec8872d3:C 2024-11-25T12:58:39,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742068_1244 (size=14341) 2024-11-25T12:58:39,910 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/75b76a041c674850b02efce9543325ce 2024-11-25T12:58:39,931 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/4d6df410dbbf4b8da03e5ef6d6272fc6 is 50, key is test_row_0/B:col10/1732539519519/Put/seqid=0 2024-11-25T12:58:39,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742069_1245 (size=12001) 2024-11-25T12:58:39,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:39,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53022 deadline: 1732539579990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:39,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:39,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1732539579990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:39,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:39,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:39,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:39,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:39,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:39,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:39,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:39,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53014 deadline: 1732539579990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:39,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:39,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:39,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:39,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:39,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52942 deadline: 1732539579990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:39,993 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:39,993 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:39,993 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:39,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:39,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52996 deadline: 1732539579990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:39,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:39,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:39,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:40,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-25T12:58:40,015 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-25T12:58:40,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:40,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. as already flushing 2024-11-25T12:58:40,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:40,015 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:40,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:40,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:40,167 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-25T12:58:40,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:40,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. as already flushing 2024-11-25T12:58:40,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:40,168 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:40,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:40,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:40,206 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:40,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53022 deadline: 1732539580204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:40,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:40,206 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:40,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1732539580204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:40,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53014 deadline: 1732539580204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:40,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:40,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52942 deadline: 1732539580204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:40,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:40,207 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:40,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:40,207 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:40,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:40,207 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:40,207 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:40,207 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:40,207 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:40,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:40,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52996 deadline: 1732539580204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:40,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:40,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:40,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:40,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:40,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:40,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:40,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-25T12:58:40,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:40,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. as already flushing 2024-11-25T12:58:40,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:40,324 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:40,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:40,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:40,370 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/4d6df410dbbf4b8da03e5ef6d6272fc6 2024-11-25T12:58:40,393 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/bf065fa4665d405090357f1f36923745 is 50, key is test_row_0/C:col10/1732539519519/Put/seqid=0 2024-11-25T12:58:40,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742070_1246 (size=12001) 2024-11-25T12:58:40,477 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-25T12:58:40,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:40,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. as already flushing 2024-11-25T12:58:40,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:40,477 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:40,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:40,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:40,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:40,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1732539580510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:40,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:40,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53022 deadline: 1732539580510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:40,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:40,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:40,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:40,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:40,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:40,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:40,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-25T12:58:40,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:40,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52942 deadline: 1732539580520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:40,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:40,523 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:40,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53014 deadline: 1732539580520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:40,523 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:40,523 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:40,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:40,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52996 deadline: 1732539580521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:40,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:40,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:40,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:40,523 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:40,523 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:40,523 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:40,631 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-25T12:58:40,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:40,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. as already flushing 2024-11-25T12:58:40,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:40,631 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:40,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:40,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:40,784 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-25T12:58:40,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:40,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. as already flushing 2024-11-25T12:58:40,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:40,784 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:40,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:40,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:40,834 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/bf065fa4665d405090357f1f36923745 2024-11-25T12:58:40,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/75b76a041c674850b02efce9543325ce as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/75b76a041c674850b02efce9543325ce 2024-11-25T12:58:40,848 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/75b76a041c674850b02efce9543325ce, entries=200, sequenceid=115, filesize=14.0 K 2024-11-25T12:58:40,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/4d6df410dbbf4b8da03e5ef6d6272fc6 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/4d6df410dbbf4b8da03e5ef6d6272fc6 2024-11-25T12:58:40,858 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/4d6df410dbbf4b8da03e5ef6d6272fc6, entries=150, sequenceid=115, filesize=11.7 K 2024-11-25T12:58:40,863 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/bf065fa4665d405090357f1f36923745 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/bf065fa4665d405090357f1f36923745 2024-11-25T12:58:40,889 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/bf065fa4665d405090357f1f36923745, entries=150, sequenceid=115, filesize=11.7 K 2024-11-25T12:58:40,900 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 21ea60aa3027cec780fb8ce3ec8872d3 in 1055ms, sequenceid=115, compaction requested=false 2024-11-25T12:58:40,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 21ea60aa3027cec780fb8ce3ec8872d3: 2024-11-25T12:58:40,951 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-25T12:58:40,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:40,952 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing 21ea60aa3027cec780fb8ce3ec8872d3 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-25T12:58:40,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/6aba27c71ffd4303a77806ca7a0b2dc1 is 50, key is test_row_0/A:col10/1732539519875/Put/seqid=0 2024-11-25T12:58:41,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742071_1247 (size=12101) 2024-11-25T12:58:41,008 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/6aba27c71ffd4303a77806ca7a0b2dc1 2024-11-25T12:58:41,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/d5e560ec7b2b4abd8c4df82cd0df8019 is 50, key is test_row_0/B:col10/1732539519875/Put/seqid=0 2024-11-25T12:58:41,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 21ea60aa3027cec780fb8ce3ec8872d3 2024-11-25T12:58:41,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. as already flushing 2024-11-25T12:58:41,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742072_1248 (size=12101) 2024-11-25T12:58:41,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:41,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53014 deadline: 1732539581042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:41,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:41,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52942 deadline: 1732539581041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:41,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:41,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:41,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:41,050 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:41,050 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:41,050 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:41,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:41,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1732539581047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:41,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:41,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53022 deadline: 1732539581048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:41,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:41,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:41,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:41,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:41,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:41,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:41,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:41,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52996 deadline: 1732539581049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:41,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:41,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:41,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:41,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:41,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53014 deadline: 1732539581159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:41,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:41,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:41,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:41,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:41,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52942 deadline: 1732539581160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:41,162 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:41,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:41,162 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:41,162 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:41,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52996 deadline: 1732539581160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:41,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:41,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:41,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:41,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:41,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1732539581160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:41,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:41,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:41,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:41,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:41,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53022 deadline: 1732539581160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:41,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:41,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:41,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:41,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:41,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53014 deadline: 1732539581369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:41,372 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:41,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52996 deadline: 1732539581369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:41,372 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:41,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:41,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:41,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1732539581370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:41,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:41,373 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:41,373 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:41,373 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:41,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:41,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:41,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:41,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:41,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53022 deadline: 1732539581370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:41,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:41,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:41,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:41,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:41,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52942 deadline: 1732539581370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:41,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:41,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:41,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:41,426 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/d5e560ec7b2b4abd8c4df82cd0df8019 2024-11-25T12:58:41,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/819802ee4c744f3492f71ead70041616 is 50, key is test_row_0/C:col10/1732539519875/Put/seqid=0 2024-11-25T12:58:41,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742073_1249 (size=12101) 2024-11-25T12:58:41,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-25T12:58:41,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:41,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53014 deadline: 1732539581680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:41,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:41,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:41,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:41,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:41,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52996 deadline: 1732539581680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:41,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:41,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:41,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:41,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:41,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1732539581680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:41,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:41,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:41,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:41,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:41,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53022 deadline: 1732539581681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:41,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:41,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:41,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:41,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:41,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52942 deadline: 1732539581681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:41,684 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:41,684 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:41,684 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:41,928 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/819802ee4c744f3492f71ead70041616 2024-11-25T12:58:41,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/6aba27c71ffd4303a77806ca7a0b2dc1 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/6aba27c71ffd4303a77806ca7a0b2dc1 2024-11-25T12:58:41,962 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/6aba27c71ffd4303a77806ca7a0b2dc1, entries=150, sequenceid=132, filesize=11.8 K 2024-11-25T12:58:41,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/d5e560ec7b2b4abd8c4df82cd0df8019 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/d5e560ec7b2b4abd8c4df82cd0df8019 2024-11-25T12:58:41,998 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/d5e560ec7b2b4abd8c4df82cd0df8019, entries=150, sequenceid=132, filesize=11.8 K 2024-11-25T12:58:42,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/819802ee4c744f3492f71ead70041616 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/819802ee4c744f3492f71ead70041616 2024-11-25T12:58:42,020 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/819802ee4c744f3492f71ead70041616, entries=150, sequenceid=132, filesize=11.8 K 2024-11-25T12:58:42,026 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 21ea60aa3027cec780fb8ce3ec8872d3 in 1074ms, sequenceid=132, compaction requested=true 2024-11-25T12:58:42,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for 21ea60aa3027cec780fb8ce3ec8872d3: 2024-11-25T12:58:42,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:42,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-25T12:58:42,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-25T12:58:42,044 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-25T12:58:42,044 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6470 sec 2024-11-25T12:58:42,048 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 2.6610 sec 2024-11-25T12:58:42,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 21ea60aa3027cec780fb8ce3ec8872d3 2024-11-25T12:58:42,193 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:58:42,193 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 21ea60aa3027cec780fb8ce3ec8872d3 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-25T12:58:42,198 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/2d800f37ccef494d805b39462dec1f19 is 50, key is test_row_0/A:col10/1732539521046/Put/seqid=0 2024-11-25T12:58:42,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742074_1250 (size=16931) 2024-11-25T12:58:42,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:42,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:42,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1732539582208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:42,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53022 deadline: 1732539582208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:42,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:42,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52942 deadline: 1732539582209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:42,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:42,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:42,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:42,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:42,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53014 deadline: 1732539582209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:42,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:42,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:42,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:42,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:42,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52996 deadline: 1732539582210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:42,215 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:42,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:42,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:42,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:42,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:42,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:42,216 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:42,216 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:42,216 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:42,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:42,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53022 deadline: 1732539582320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:42,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:42,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:42,323 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:42,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53014 deadline: 1732539582320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:42,323 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:42,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:42,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:42,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:42,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:42,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52996 deadline: 1732539582320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:42,323 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:42,323 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:42,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:42,323 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:42,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52942 deadline: 1732539582320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:42,323 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:42,323 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:42,323 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:42,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:42,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1732539582320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:42,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:42,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:42,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:42,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:42,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53022 deadline: 1732539582530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:42,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:42,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:42,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:42,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:42,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52996 deadline: 1732539582531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:42,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:42,533 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:42,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:42,533 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:42,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52942 deadline: 1732539582531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:42,533 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:42,533 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:42,533 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:42,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:42,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1732539582531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:42,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:42,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:42,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:42,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:42,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53014 deadline: 1732539582531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:42,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:42,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:42,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:42,605 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/2d800f37ccef494d805b39462dec1f19 2024-11-25T12:58:42,633 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/bb966ca48934440e9b0df545d41485d2 is 50, key is test_row_0/B:col10/1732539521046/Put/seqid=0 2024-11-25T12:58:42,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742075_1251 (size=12151) 2024-11-25T12:58:42,675 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/bb966ca48934440e9b0df545d41485d2 2024-11-25T12:58:42,693 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/006162a661c1447c94528798ca6c200b is 50, key is test_row_0/C:col10/1732539521046/Put/seqid=0 2024-11-25T12:58:42,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742076_1252 (size=12151) 2024-11-25T12:58:42,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:42,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53022 deadline: 1732539582840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:42,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:42,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:42,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:42,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:42,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52996 deadline: 1732539582841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:42,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:42,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:42,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:42,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:42,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53014 deadline: 1732539582841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:42,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:42,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:42,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:42,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:42,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52942 deadline: 1732539582841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:42,848 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:42,848 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:42,848 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:42,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:42,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1732539582841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:42,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:42,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:42,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:43,118 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/006162a661c1447c94528798ca6c200b 2024-11-25T12:58:43,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/2d800f37ccef494d805b39462dec1f19 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/2d800f37ccef494d805b39462dec1f19 2024-11-25T12:58:43,135 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/2d800f37ccef494d805b39462dec1f19, entries=250, sequenceid=154, filesize=16.5 K 2024-11-25T12:58:43,136 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/bb966ca48934440e9b0df545d41485d2 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/bb966ca48934440e9b0df545d41485d2 2024-11-25T12:58:43,152 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/bb966ca48934440e9b0df545d41485d2, entries=150, sequenceid=154, filesize=11.9 K 2024-11-25T12:58:43,153 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/006162a661c1447c94528798ca6c200b as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/006162a661c1447c94528798ca6c200b 2024-11-25T12:58:43,180 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/006162a661c1447c94528798ca6c200b, entries=150, sequenceid=154, filesize=11.9 K 2024-11-25T12:58:43,185 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=87.22 KB/89310 for 21ea60aa3027cec780fb8ce3ec8872d3 in 992ms, sequenceid=154, compaction requested=true 2024-11-25T12:58:43,185 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 21ea60aa3027cec780fb8ce3ec8872d3: 2024-11-25T12:58:43,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21ea60aa3027cec780fb8ce3ec8872d3:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:58:43,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:58:43,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21ea60aa3027cec780fb8ce3ec8872d3:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:58:43,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-25T12:58:43,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21ea60aa3027cec780fb8ce3ec8872d3:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:58:43,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-25T12:58:43,186 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:58:43,186 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:58:43,189 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48460 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:58:43,189 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): 21ea60aa3027cec780fb8ce3ec8872d3/C is initiating minor compaction (all files) 2024-11-25T12:58:43,189 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21ea60aa3027cec780fb8ce3ec8872d3/C in TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:43,189 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/2ffb2f8c2f044c2581207e18c41902d5, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/bf065fa4665d405090357f1f36923745, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/819802ee4c744f3492f71ead70041616, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/006162a661c1447c94528798ca6c200b] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp, totalSize=47.3 K 2024-11-25T12:58:43,191 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55580 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:58:43,191 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1541): 21ea60aa3027cec780fb8ce3ec8872d3/A is initiating minor compaction (all files) 2024-11-25T12:58:43,191 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21ea60aa3027cec780fb8ce3ec8872d3/A in TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:43,191 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/34340069d0e043d4ac4ab64128a808f8, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/75b76a041c674850b02efce9543325ce, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/6aba27c71ffd4303a77806ca7a0b2dc1, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/2d800f37ccef494d805b39462dec1f19] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp, totalSize=54.3 K 2024-11-25T12:58:43,191 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ffb2f8c2f044c2581207e18c41902d5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732539518198 2024-11-25T12:58:43,192 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34340069d0e043d4ac4ab64128a808f8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732539518198 2024-11-25T12:58:43,192 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting bf065fa4665d405090357f1f36923745, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732539519488 2024-11-25T12:58:43,192 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75b76a041c674850b02efce9543325ce, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732539519488 2024-11-25T12:58:43,192 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 819802ee4c744f3492f71ead70041616, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732539519872 2024-11-25T12:58:43,192 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6aba27c71ffd4303a77806ca7a0b2dc1, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732539519872 2024-11-25T12:58:43,193 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 006162a661c1447c94528798ca6c200b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732539521046 2024-11-25T12:58:43,193 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d800f37ccef494d805b39462dec1f19, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732539521039 2024-11-25T12:58:43,208 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21ea60aa3027cec780fb8ce3ec8872d3#A#compaction#207 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:58:43,209 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/2faf7ea9433641149a5cbe7122dfdae3 is 50, key is test_row_0/A:col10/1732539521046/Put/seqid=0 2024-11-25T12:58:43,215 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21ea60aa3027cec780fb8ce3ec8872d3#C#compaction#208 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:58:43,216 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/686367e550324a5c95b396a31f59078a is 50, key is test_row_0/C:col10/1732539521046/Put/seqid=0 2024-11-25T12:58:43,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742077_1253 (size=12493) 2024-11-25T12:58:43,260 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/2faf7ea9433641149a5cbe7122dfdae3 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/2faf7ea9433641149a5cbe7122dfdae3 2024-11-25T12:58:43,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742078_1254 (size=12493) 2024-11-25T12:58:43,279 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/686367e550324a5c95b396a31f59078a as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/686367e550324a5c95b396a31f59078a 2024-11-25T12:58:43,282 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 21ea60aa3027cec780fb8ce3ec8872d3/A of 21ea60aa3027cec780fb8ce3ec8872d3 into 2faf7ea9433641149a5cbe7122dfdae3(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:58:43,282 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21ea60aa3027cec780fb8ce3ec8872d3: 2024-11-25T12:58:43,282 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., storeName=21ea60aa3027cec780fb8ce3ec8872d3/A, priority=12, startTime=1732539523185; duration=0sec 2024-11-25T12:58:43,282 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:58:43,282 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21ea60aa3027cec780fb8ce3ec8872d3:A 2024-11-25T12:58:43,282 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:58:43,287 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48460 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:58:43,287 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1541): 21ea60aa3027cec780fb8ce3ec8872d3/B is initiating minor compaction (all files) 2024-11-25T12:58:43,287 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21ea60aa3027cec780fb8ce3ec8872d3/B in TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:43,287 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/44e1280f7b5c47988c86898d9492e6b1, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/4d6df410dbbf4b8da03e5ef6d6272fc6, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/d5e560ec7b2b4abd8c4df82cd0df8019, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/bb966ca48934440e9b0df545d41485d2] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp, totalSize=47.3 K 2024-11-25T12:58:43,293 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44e1280f7b5c47988c86898d9492e6b1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732539518198 2024-11-25T12:58:43,293 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d6df410dbbf4b8da03e5ef6d6272fc6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732539519488 2024-11-25T12:58:43,294 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5e560ec7b2b4abd8c4df82cd0df8019, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732539519872 2024-11-25T12:58:43,294 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb966ca48934440e9b0df545d41485d2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732539521046 2024-11-25T12:58:43,296 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 21ea60aa3027cec780fb8ce3ec8872d3/C of 21ea60aa3027cec780fb8ce3ec8872d3 into 686367e550324a5c95b396a31f59078a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:58:43,296 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21ea60aa3027cec780fb8ce3ec8872d3: 2024-11-25T12:58:43,296 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., storeName=21ea60aa3027cec780fb8ce3ec8872d3/C, priority=12, startTime=1732539523185; duration=0sec 2024-11-25T12:58:43,297 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:58:43,297 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21ea60aa3027cec780fb8ce3ec8872d3:C 2024-11-25T12:58:43,326 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21ea60aa3027cec780fb8ce3ec8872d3#B#compaction#209 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:58:43,327 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/90725d57a4534cc18352a65d81f5b621 is 50, key is test_row_0/B:col10/1732539521046/Put/seqid=0 2024-11-25T12:58:43,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742079_1255 (size=12493) 2024-11-25T12:58:43,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 21ea60aa3027cec780fb8ce3ec8872d3 2024-11-25T12:58:43,372 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:58:43,372 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 21ea60aa3027cec780fb8ce3ec8872d3 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-25T12:58:43,383 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/90725d57a4534cc18352a65d81f5b621 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/90725d57a4534cc18352a65d81f5b621 2024-11-25T12:58:43,391 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/ae5fd395d52c49629cb33a5eddb474be is 50, key is test_row_0/A:col10/1732539523372/Put/seqid=0 2024-11-25T12:58:43,403 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 21ea60aa3027cec780fb8ce3ec8872d3/B of 21ea60aa3027cec780fb8ce3ec8872d3 into 90725d57a4534cc18352a65d81f5b621(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:58:43,403 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21ea60aa3027cec780fb8ce3ec8872d3: 2024-11-25T12:58:43,403 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., storeName=21ea60aa3027cec780fb8ce3ec8872d3/B, priority=12, startTime=1732539523185; duration=0sec 2024-11-25T12:58:43,404 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:58:43,404 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21ea60aa3027cec780fb8ce3ec8872d3:B 2024-11-25T12:58:43,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742080_1256 (size=14541) 2024-11-25T12:58:43,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:43,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53022 deadline: 1732539583433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:43,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:43,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:43,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:43,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:43,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52996 deadline: 1732539583440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:43,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:43,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53014 deadline: 1732539583446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:43,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:43,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52942 deadline: 1732539583447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:43,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:43,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:43,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:43,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:43,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1732539583456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:43,461 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:43,461 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:43,461 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:43,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:43,463 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:43,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:43,463 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:43,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:43,463 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:43,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-25T12:58:43,550 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:58:43,551 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:58:43,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-25T12:58:43,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-25T12:58:43,563 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:58:43,564 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:58:43,564 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:58:43,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:43,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53022 deadline: 1732539583560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:43,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:43,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:43,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:43,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:43,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52996 deadline: 1732539583571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:43,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:43,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52942 deadline: 1732539583571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:43,575 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:43,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:43,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:43,575 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:43,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:43,575 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:43,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:43,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53014 deadline: 1732539583571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:43,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:43,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:43,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:43,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:43,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1732539583576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:43,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:43,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:43,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:43,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-25T12:58:43,720 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-25T12:58:43,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:43,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. as already flushing 2024-11-25T12:58:43,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:43,720 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:43,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:43,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:43,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:43,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53022 deadline: 1732539583770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:43,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:43,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:43,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:43,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:43,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52942 deadline: 1732539583780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:43,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:43,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52996 deadline: 1732539583780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:43,782 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:43,782 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:43,782 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:43,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:43,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:43,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:43,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:43,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53014 deadline: 1732539583790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:43,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:43,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:43,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:43,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:43,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1732539583800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:43,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:43,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:43,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:43,828 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/ae5fd395d52c49629cb33a5eddb474be 2024-11-25T12:58:43,835 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/994b5e0d99ab449d98d613e1d2dffa89 is 50, key is test_row_0/B:col10/1732539523372/Put/seqid=0 2024-11-25T12:58:43,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742081_1257 (size=12151) 2024-11-25T12:58:43,845 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/994b5e0d99ab449d98d613e1d2dffa89 2024-11-25T12:58:43,853 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/8f002ff1b2174d8882df3e86be63285f is 50, key is test_row_0/C:col10/1732539523372/Put/seqid=0 2024-11-25T12:58:43,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-25T12:58:43,873 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-25T12:58:43,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:43,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. as already flushing 2024-11-25T12:58:43,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:43,873 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:43,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:43,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:43,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742082_1258 (size=12151) 2024-11-25T12:58:43,895 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/8f002ff1b2174d8882df3e86be63285f 2024-11-25T12:58:43,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/ae5fd395d52c49629cb33a5eddb474be as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/ae5fd395d52c49629cb33a5eddb474be 2024-11-25T12:58:43,919 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/ae5fd395d52c49629cb33a5eddb474be, entries=200, sequenceid=175, filesize=14.2 K 2024-11-25T12:58:43,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/994b5e0d99ab449d98d613e1d2dffa89 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/994b5e0d99ab449d98d613e1d2dffa89 2024-11-25T12:58:43,926 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/994b5e0d99ab449d98d613e1d2dffa89, entries=150, sequenceid=175, filesize=11.9 K 2024-11-25T12:58:43,927 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/8f002ff1b2174d8882df3e86be63285f as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/8f002ff1b2174d8882df3e86be63285f 2024-11-25T12:58:43,931 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/8f002ff1b2174d8882df3e86be63285f, entries=150, sequenceid=175, filesize=11.9 K 2024-11-25T12:58:43,932 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 21ea60aa3027cec780fb8ce3ec8872d3 in 560ms, sequenceid=175, compaction requested=false 2024-11-25T12:58:43,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 21ea60aa3027cec780fb8ce3ec8872d3: 2024-11-25T12:58:44,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-25T12:58:44,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:44,029 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 21ea60aa3027cec780fb8ce3ec8872d3 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-25T12:58:44,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/c4a9f0cc8814487a8bcf5e725f03ee01 is 50, key is test_row_0/A:col10/1732539523422/Put/seqid=0 2024-11-25T12:58:44,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 21ea60aa3027cec780fb8ce3ec8872d3 2024-11-25T12:58:44,083 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. as already flushing 2024-11-25T12:58:44,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742083_1259 (size=12151) 2024-11-25T12:58:44,112 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/c4a9f0cc8814487a8bcf5e725f03ee01 2024-11-25T12:58:44,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:44,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52996 deadline: 1732539584137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:44,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:44,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:44,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:44,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:44,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53022 deadline: 1732539584144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:44,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:44,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:44,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:44,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/9e21995419c34f8d9772ab9ae5311e65 is 50, key is test_row_0/B:col10/1732539523422/Put/seqid=0 2024-11-25T12:58:44,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:44,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53014 deadline: 1732539584147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:44,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:44,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1732539584148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:44,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:44,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:44,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:44,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:44,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52942 deadline: 1732539584151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:44,163 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:44,163 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:44,163 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:44,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:44,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:44,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:44,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-25T12:58:44,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742084_1260 (size=12151) 2024-11-25T12:58:44,239 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/9e21995419c34f8d9772ab9ae5311e65 2024-11-25T12:58:44,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:44,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52996 deadline: 1732539584250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:44,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:44,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:44,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:44,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:44,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53022 deadline: 1732539584260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:44,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:44,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:44,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:44,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:44,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53014 deadline: 1732539584270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:44,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:44,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52942 deadline: 1732539584271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:44,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:44,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1732539584271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:44,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:44,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:44,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:44,275 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:44,275 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:44,275 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:44,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:44,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:44,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:44,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/651da5d107c848599d6d44553bb82389 is 50, key is test_row_0/C:col10/1732539523422/Put/seqid=0 2024-11-25T12:58:44,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742085_1261 (size=12151) 2024-11-25T12:58:44,329 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/651da5d107c848599d6d44553bb82389 2024-11-25T12:58:44,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/c4a9f0cc8814487a8bcf5e725f03ee01 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/c4a9f0cc8814487a8bcf5e725f03ee01 2024-11-25T12:58:44,339 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/c4a9f0cc8814487a8bcf5e725f03ee01, entries=150, sequenceid=193, filesize=11.9 K 2024-11-25T12:58:44,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/9e21995419c34f8d9772ab9ae5311e65 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/9e21995419c34f8d9772ab9ae5311e65 2024-11-25T12:58:44,346 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/9e21995419c34f8d9772ab9ae5311e65, entries=150, sequenceid=193, filesize=11.9 K 2024-11-25T12:58:44,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/651da5d107c848599d6d44553bb82389 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/651da5d107c848599d6d44553bb82389 2024-11-25T12:58:44,353 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/651da5d107c848599d6d44553bb82389, entries=150, sequenceid=193, filesize=11.9 K 2024-11-25T12:58:44,355 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=114.05 KB/116790 for 21ea60aa3027cec780fb8ce3ec8872d3 in 326ms, sequenceid=193, compaction requested=true 2024-11-25T12:58:44,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 21ea60aa3027cec780fb8ce3ec8872d3: 2024-11-25T12:58:44,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:44,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-25T12:58:44,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-25T12:58:44,359 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-25T12:58:44,359 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 793 msec 2024-11-25T12:58:44,363 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 809 msec 2024-11-25T12:58:44,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 21ea60aa3027cec780fb8ce3ec8872d3 2024-11-25T12:58:44,473 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:58:44,473 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 21ea60aa3027cec780fb8ce3ec8872d3 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-25T12:58:44,490 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/4601ffae07694a84b04aa5ed1f453620 is 50, key is test_row_0/A:col10/1732539524147/Put/seqid=0 2024-11-25T12:58:44,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:44,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53014 deadline: 1732539584505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:44,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:44,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:44,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:44,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:44,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52996 deadline: 1732539584512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:44,523 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:44,524 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:44,524 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:44,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:44,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1732539584513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:44,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:44,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:44,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:44,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:44,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52942 deadline: 1732539584517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:44,533 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:44,533 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:44,533 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:44,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:44,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53022 deadline: 1732539584515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:44,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:44,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:44,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:44,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742086_1262 (size=14541) 2024-11-25T12:58:44,560 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/4601ffae07694a84b04aa5ed1f453620 2024-11-25T12:58:44,575 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/b80b6c887965448d9ef0deedb3f2d717 is 50, key is test_row_0/B:col10/1732539524147/Put/seqid=0 2024-11-25T12:58:44,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742087_1263 (size=12151) 2024-11-25T12:58:44,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:44,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53014 deadline: 1732539584620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:44,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:44,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:44,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:44,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:44,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52996 deadline: 1732539584630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:44,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:44,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:44,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:44,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:44,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1732539584630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:44,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:44,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:44,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:44,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:44,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53022 deadline: 1732539584640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:44,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:44,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:44,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:44,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:44,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52942 deadline: 1732539584640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:44,655 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:44,655 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:44,655 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:44,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-25T12:58:44,692 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:58:44,692 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:58:44,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-25T12:58:44,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-25T12:58:44,694 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:58:44,695 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:58:44,695 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:58:44,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-25T12:58:44,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:44,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53014 deadline: 1732539584831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:44,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:44,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:44,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:44,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:44,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52996 deadline: 1732539584841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:44,849 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-25T12:58:44,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:44,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. as already flushing 2024-11-25T12:58:44,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:44,849 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:44,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:44,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:44,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:44,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:44,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:44,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:44,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1732539584850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:44,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:44,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:44,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:44,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:44,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53022 deadline: 1732539584866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:44,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:44,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:44,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:44,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:44,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52942 deadline: 1732539584866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:44,875 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:44,875 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:44,875 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:45,002 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-25T12:58:45,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:45,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. as already flushing 2024-11-25T12:58:45,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:45,003 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:45,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:45,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:45,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-25T12:58:45,022 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/b80b6c887965448d9ef0deedb3f2d717 2024-11-25T12:58:45,037 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/6442bf90c5364fd2ae6185b72d8debd8 is 50, key is test_row_0/C:col10/1732539524147/Put/seqid=0 2024-11-25T12:58:45,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742088_1264 (size=12151) 2024-11-25T12:58:45,077 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/6442bf90c5364fd2ae6185b72d8debd8 2024-11-25T12:58:45,091 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/4601ffae07694a84b04aa5ed1f453620 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/4601ffae07694a84b04aa5ed1f453620 2024-11-25T12:58:45,112 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/4601ffae07694a84b04aa5ed1f453620, entries=200, sequenceid=214, filesize=14.2 K 2024-11-25T12:58:45,115 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/b80b6c887965448d9ef0deedb3f2d717 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/b80b6c887965448d9ef0deedb3f2d717 2024-11-25T12:58:45,121 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/b80b6c887965448d9ef0deedb3f2d717, entries=150, sequenceid=214, filesize=11.9 K 2024-11-25T12:58:45,122 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/6442bf90c5364fd2ae6185b72d8debd8 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/6442bf90c5364fd2ae6185b72d8debd8 2024-11-25T12:58:45,128 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/6442bf90c5364fd2ae6185b72d8debd8, entries=150, sequenceid=214, filesize=11.9 K 2024-11-25T12:58:45,129 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 21ea60aa3027cec780fb8ce3ec8872d3 in 655ms, sequenceid=214, compaction requested=true 2024-11-25T12:58:45,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 21ea60aa3027cec780fb8ce3ec8872d3: 2024-11-25T12:58:45,129 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:58:45,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21ea60aa3027cec780fb8ce3ec8872d3:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:58:45,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:58:45,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21ea60aa3027cec780fb8ce3ec8872d3:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:58:45,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:58:45,130 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:58:45,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21ea60aa3027cec780fb8ce3ec8872d3:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:58:45,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:58:45,131 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53726 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:58:45,131 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1541): 21ea60aa3027cec780fb8ce3ec8872d3/A is initiating minor compaction (all files) 2024-11-25T12:58:45,131 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21ea60aa3027cec780fb8ce3ec8872d3/A in TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:45,131 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/2faf7ea9433641149a5cbe7122dfdae3, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/ae5fd395d52c49629cb33a5eddb474be, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/c4a9f0cc8814487a8bcf5e725f03ee01, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/4601ffae07694a84b04aa5ed1f453620] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp, totalSize=52.5 K 2024-11-25T12:58:45,132 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2faf7ea9433641149a5cbe7122dfdae3, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732539521046 2024-11-25T12:58:45,132 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae5fd395d52c49629cb33a5eddb474be, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1732539522207 2024-11-25T12:58:45,140 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:58:45,140 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): 21ea60aa3027cec780fb8ce3ec8872d3/B is initiating minor compaction (all files) 2024-11-25T12:58:45,140 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21ea60aa3027cec780fb8ce3ec8872d3/B in TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:45,140 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/90725d57a4534cc18352a65d81f5b621, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/994b5e0d99ab449d98d613e1d2dffa89, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/9e21995419c34f8d9772ab9ae5311e65, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/b80b6c887965448d9ef0deedb3f2d717] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp, totalSize=47.8 K 2024-11-25T12:58:45,141 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4a9f0cc8814487a8bcf5e725f03ee01, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732539523422 2024-11-25T12:58:45,141 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 90725d57a4534cc18352a65d81f5b621, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732539521046 2024-11-25T12:58:45,145 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4601ffae07694a84b04aa5ed1f453620, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732539524141 2024-11-25T12:58:45,147 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 994b5e0d99ab449d98d613e1d2dffa89, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1732539523365 2024-11-25T12:58:45,148 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e21995419c34f8d9772ab9ae5311e65, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732539523422 2024-11-25T12:58:45,149 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting b80b6c887965448d9ef0deedb3f2d717, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732539524143 2024-11-25T12:58:45,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-25T12:58:45,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:45,164 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 21ea60aa3027cec780fb8ce3ec8872d3 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-25T12:58:45,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 21ea60aa3027cec780fb8ce3ec8872d3 2024-11-25T12:58:45,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. as already flushing 2024-11-25T12:58:45,177 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21ea60aa3027cec780fb8ce3ec8872d3#A#compaction#219 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:58:45,178 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/745f21900c6846b18f971a4449f2332c is 50, key is test_row_0/A:col10/1732539524147/Put/seqid=0 2024-11-25T12:58:45,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/f25ec1eae4064f318581c4559336d8ff is 50, key is test_row_0/A:col10/1732539524512/Put/seqid=0 2024-11-25T12:58:45,204 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21ea60aa3027cec780fb8ce3ec8872d3#B#compaction#221 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:58:45,205 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/cb857945b73044d1b14552c1b21c5479 is 50, key is test_row_0/B:col10/1732539524147/Put/seqid=0 2024-11-25T12:58:45,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742089_1265 (size=12629) 2024-11-25T12:58:45,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:45,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52996 deadline: 1732539585221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:45,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:45,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:45,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:45,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:45,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53022 deadline: 1732539585223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:45,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:45,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:45,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:45,231 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/745f21900c6846b18f971a4449f2332c as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/745f21900c6846b18f971a4449f2332c 2024-11-25T12:58:45,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:45,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53014 deadline: 1732539585227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:45,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:45,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:45,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:45,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:45,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1732539585229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:45,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:45,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:45,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:45,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:45,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52942 deadline: 1732539585229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:45,238 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:45,238 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:45,238 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:45,238 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 21ea60aa3027cec780fb8ce3ec8872d3/A of 21ea60aa3027cec780fb8ce3ec8872d3 into 745f21900c6846b18f971a4449f2332c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:58:45,238 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21ea60aa3027cec780fb8ce3ec8872d3: 2024-11-25T12:58:45,238 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., storeName=21ea60aa3027cec780fb8ce3ec8872d3/A, priority=12, startTime=1732539525129; duration=0sec 2024-11-25T12:58:45,239 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:58:45,239 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21ea60aa3027cec780fb8ce3ec8872d3:A 2024-11-25T12:58:45,239 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:58:45,243 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:58:45,243 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1541): 21ea60aa3027cec780fb8ce3ec8872d3/C is initiating minor compaction (all files) 2024-11-25T12:58:45,243 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21ea60aa3027cec780fb8ce3ec8872d3/C in TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:45,243 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/686367e550324a5c95b396a31f59078a, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/8f002ff1b2174d8882df3e86be63285f, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/651da5d107c848599d6d44553bb82389, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/6442bf90c5364fd2ae6185b72d8debd8] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp, totalSize=47.8 K 2024-11-25T12:58:45,244 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 686367e550324a5c95b396a31f59078a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732539521046 2024-11-25T12:58:45,246 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f002ff1b2174d8882df3e86be63285f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1732539523365 2024-11-25T12:58:45,246 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 651da5d107c848599d6d44553bb82389, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732539523422 2024-11-25T12:58:45,246 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6442bf90c5364fd2ae6185b72d8debd8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732539524143 2024-11-25T12:58:45,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742090_1266 (size=14541) 2024-11-25T12:58:45,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742091_1267 (size=12629) 2024-11-25T12:58:45,290 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/cb857945b73044d1b14552c1b21c5479 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/cb857945b73044d1b14552c1b21c5479 2024-11-25T12:58:45,294 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21ea60aa3027cec780fb8ce3ec8872d3#C#compaction#222 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:58:45,294 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/126258b976a147e8b9c8fdcf5281bab5 is 50, key is test_row_0/C:col10/1732539524147/Put/seqid=0 2024-11-25T12:58:45,309 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 21ea60aa3027cec780fb8ce3ec8872d3/B of 21ea60aa3027cec780fb8ce3ec8872d3 into cb857945b73044d1b14552c1b21c5479(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:58:45,310 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21ea60aa3027cec780fb8ce3ec8872d3: 2024-11-25T12:58:45,310 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., storeName=21ea60aa3027cec780fb8ce3ec8872d3/B, priority=12, startTime=1732539525130; duration=0sec 2024-11-25T12:58:45,310 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:58:45,310 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21ea60aa3027cec780fb8ce3ec8872d3:B 2024-11-25T12:58:45,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-25T12:58:45,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742092_1268 (size=12629) 2024-11-25T12:58:45,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:45,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52996 deadline: 1732539585340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:45,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:45,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:45,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:45,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:45,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53022 deadline: 1732539585340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:45,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:45,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:45,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:45,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:45,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1732539585340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:45,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:45,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:45,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:45,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:45,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53014 deadline: 1732539585340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:45,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:45,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:45,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:45,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:45,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52942 deadline: 1732539585343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:45,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:45,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:45,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:45,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:45,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52996 deadline: 1732539585552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:45,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:45,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:45,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:45,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1732539585552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:45,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:45,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:45,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:45,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:45,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:45,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53014 deadline: 1732539585552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:45,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:45,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:45,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:45,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:45,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53022 deadline: 1732539585552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:45,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:45,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:45,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:45,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:45,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52942 deadline: 1732539585563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:45,572 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:45,572 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:45,572 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:45,664 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/f25ec1eae4064f318581c4559336d8ff 2024-11-25T12:58:45,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/1255251adaf94efc959aa997bbd12426 is 50, key is test_row_0/B:col10/1732539524512/Put/seqid=0 2024-11-25T12:58:45,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742093_1269 (size=12151) 2024-11-25T12:58:45,770 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/126258b976a147e8b9c8fdcf5281bab5 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/126258b976a147e8b9c8fdcf5281bab5 2024-11-25T12:58:45,776 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 21ea60aa3027cec780fb8ce3ec8872d3/C of 21ea60aa3027cec780fb8ce3ec8872d3 into 126258b976a147e8b9c8fdcf5281bab5(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:58:45,776 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21ea60aa3027cec780fb8ce3ec8872d3: 2024-11-25T12:58:45,776 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., storeName=21ea60aa3027cec780fb8ce3ec8872d3/C, priority=12, startTime=1732539525130; duration=0sec 2024-11-25T12:58:45,777 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:58:45,777 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21ea60aa3027cec780fb8ce3ec8872d3:C 2024-11-25T12:58:45,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-25T12:58:45,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:45,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52996 deadline: 1732539585860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:45,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:45,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:45,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:45,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:45,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53022 deadline: 1732539585870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:45,871 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:45,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53014 deadline: 1732539585870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:45,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:45,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:45,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:45,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:45,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:45,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:45,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:45,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1732539585872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:45,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:45,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:45,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:45,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:45,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52942 deadline: 1732539585885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:45,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:45,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:45,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:46,162 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/1255251adaf94efc959aa997bbd12426 2024-11-25T12:58:46,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/b5477000f70c477fbb4283c04ac1e3e5 is 50, key is test_row_0/C:col10/1732539524512/Put/seqid=0 2024-11-25T12:58:46,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742094_1270 (size=12151) 2024-11-25T12:58:46,194 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/b5477000f70c477fbb4283c04ac1e3e5 2024-11-25T12:58:46,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/f25ec1eae4064f318581c4559336d8ff as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/f25ec1eae4064f318581c4559336d8ff 2024-11-25T12:58:46,216 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/f25ec1eae4064f318581c4559336d8ff, entries=200, sequenceid=230, filesize=14.2 K 2024-11-25T12:58:46,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/1255251adaf94efc959aa997bbd12426 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/1255251adaf94efc959aa997bbd12426 2024-11-25T12:58:46,227 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/1255251adaf94efc959aa997bbd12426, entries=150, sequenceid=230, filesize=11.9 K 2024-11-25T12:58:46,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/b5477000f70c477fbb4283c04ac1e3e5 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/b5477000f70c477fbb4283c04ac1e3e5 2024-11-25T12:58:46,236 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/b5477000f70c477fbb4283c04ac1e3e5, entries=150, sequenceid=230, filesize=11.9 K 2024-11-25T12:58:46,237 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for 21ea60aa3027cec780fb8ce3ec8872d3 in 1073ms, sequenceid=230, compaction requested=false 2024-11-25T12:58:46,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 21ea60aa3027cec780fb8ce3ec8872d3: 2024-11-25T12:58:46,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:46,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-25T12:58:46,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-25T12:58:46,241 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-25T12:58:46,241 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5440 sec 2024-11-25T12:58:46,243 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.5500 sec 2024-11-25T12:58:46,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 21ea60aa3027cec780fb8ce3ec8872d3 2024-11-25T12:58:46,379 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:58:46,380 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 21ea60aa3027cec780fb8ce3ec8872d3 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-25T12:58:46,393 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/fd95edb09a3c4ed5b5c931d68023a341 is 50, key is test_row_0/A:col10/1732539526375/Put/seqid=0 2024-11-25T12:58:46,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:46,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1732539586408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:46,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:46,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:46,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:46,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:46,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53022 deadline: 1732539586412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:46,422 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:46,422 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:46,422 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:46,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:46,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52942 deadline: 1732539586416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:46,427 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:46,427 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:46,427 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:46,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:46,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52996 deadline: 1732539586421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:46,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:46,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:46,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:46,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742095_1271 (size=14541) 2024-11-25T12:58:46,437 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/fd95edb09a3c4ed5b5c931d68023a341 2024-11-25T12:58:46,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:46,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53014 deadline: 1732539586456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:46,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:46,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:46,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:46,463 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/b562c6273f024a81ac4a40f95532dc28 is 50, key is test_row_0/B:col10/1732539526375/Put/seqid=0 2024-11-25T12:58:46,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742096_1272 (size=12151) 2024-11-25T12:58:46,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:46,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53022 deadline: 1732539586530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:46,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:46,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1732539586531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:46,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:46,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52942 deadline: 1732539586531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:46,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:46,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:46,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:46,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:46,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:46,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:46,534 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:46,534 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:46,534 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:46,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:46,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52996 deadline: 1732539586540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:46,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:46,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:46,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:46,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:46,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53014 deadline: 1732539586570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:46,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:46,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:46,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:46,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:46,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53022 deadline: 1732539586740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:46,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:46,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:46,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:46,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52942 deadline: 1732539586740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:46,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:46,743 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:46,743 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:46,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:46,743 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:46,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1732539586740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:46,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:46,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:46,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:46,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:46,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52996 deadline: 1732539586749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:46,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:46,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:46,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:46,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:46,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53014 deadline: 1732539586780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:46,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:46,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:46,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:46,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-25T12:58:46,850 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:58:46,851 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:58:46,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-25T12:58:46,853 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:58:46,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-25T12:58:46,853 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:58:46,853 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:58:46,914 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/b562c6273f024a81ac4a40f95532dc28 2024-11-25T12:58:46,937 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/022755c065b943ef91f833a5c86fd5a9 is 50, key is test_row_0/C:col10/1732539526375/Put/seqid=0 2024-11-25T12:58:46,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-25T12:58:46,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742097_1273 (size=12151) 2024-11-25T12:58:46,966 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/022755c065b943ef91f833a5c86fd5a9 2024-11-25T12:58:46,972 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/fd95edb09a3c4ed5b5c931d68023a341 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/fd95edb09a3c4ed5b5c931d68023a341 2024-11-25T12:58:46,977 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/fd95edb09a3c4ed5b5c931d68023a341, entries=200, sequenceid=255, filesize=14.2 K 2024-11-25T12:58:46,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/b562c6273f024a81ac4a40f95532dc28 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/b562c6273f024a81ac4a40f95532dc28 2024-11-25T12:58:46,983 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/b562c6273f024a81ac4a40f95532dc28, entries=150, sequenceid=255, filesize=11.9 K 2024-11-25T12:58:46,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/022755c065b943ef91f833a5c86fd5a9 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/022755c065b943ef91f833a5c86fd5a9 2024-11-25T12:58:46,995 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/022755c065b943ef91f833a5c86fd5a9, entries=150, sequenceid=255, filesize=11.9 K 2024-11-25T12:58:46,997 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for 21ea60aa3027cec780fb8ce3ec8872d3 in 617ms, sequenceid=255, compaction requested=true 2024-11-25T12:58:46,997 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 21ea60aa3027cec780fb8ce3ec8872d3: 2024-11-25T12:58:46,997 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:58:46,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21ea60aa3027cec780fb8ce3ec8872d3:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:58:46,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:58:46,997 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:58:46,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21ea60aa3027cec780fb8ce3ec8872d3:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:58:46,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:58:46,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21ea60aa3027cec780fb8ce3ec8872d3:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:58:46,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:58:46,999 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:58:46,999 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1541): 21ea60aa3027cec780fb8ce3ec8872d3/A is initiating minor compaction (all files) 2024-11-25T12:58:46,999 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21ea60aa3027cec780fb8ce3ec8872d3/A in TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:46,999 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/745f21900c6846b18f971a4449f2332c, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/f25ec1eae4064f318581c4559336d8ff, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/fd95edb09a3c4ed5b5c931d68023a341] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp, totalSize=40.7 K 2024-11-25T12:58:47,000 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:58:47,000 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): 21ea60aa3027cec780fb8ce3ec8872d3/B is initiating minor compaction (all files) 2024-11-25T12:58:47,000 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21ea60aa3027cec780fb8ce3ec8872d3/B in TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:47,000 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/cb857945b73044d1b14552c1b21c5479, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/1255251adaf94efc959aa997bbd12426, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/b562c6273f024a81ac4a40f95532dc28] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp, totalSize=36.1 K 2024-11-25T12:58:47,004 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting cb857945b73044d1b14552c1b21c5479, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732539524143 2024-11-25T12:58:47,004 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 745f21900c6846b18f971a4449f2332c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732539524143 2024-11-25T12:58:47,004 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 1255251adaf94efc959aa997bbd12426, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1732539524488 2024-11-25T12:58:47,004 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting f25ec1eae4064f318581c4559336d8ff, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1732539524488 2024-11-25T12:58:47,005 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting b562c6273f024a81ac4a40f95532dc28, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732539525213 2024-11-25T12:58:47,005 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd95edb09a3c4ed5b5c931d68023a341, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732539525213 2024-11-25T12:58:47,006 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-25T12:58:47,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:47,006 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 21ea60aa3027cec780fb8ce3ec8872d3 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-25T12:58:47,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/b27934e82ff5491e82f87e2d924b0245 is 50, key is test_row_0/A:col10/1732539526420/Put/seqid=0 2024-11-25T12:58:47,032 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21ea60aa3027cec780fb8ce3ec8872d3#A#compaction#229 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:58:47,032 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/1c2f52fa996e47d2b8db5cddfb8427d3 is 50, key is test_row_0/A:col10/1732539526375/Put/seqid=0 2024-11-25T12:58:47,041 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21ea60aa3027cec780fb8ce3ec8872d3#B#compaction#230 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:58:47,042 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/e40085c8dbec47f6912b3c4d47f97d2c is 50, key is test_row_0/B:col10/1732539526375/Put/seqid=0 2024-11-25T12:58:47,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. as already flushing 2024-11-25T12:58:47,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 21ea60aa3027cec780fb8ce3ec8872d3 2024-11-25T12:58:47,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:47,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53014 deadline: 1732539587090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:47,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:47,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53022 deadline: 1732539587106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:47,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:47,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:47,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:47,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742098_1274 (size=12301) 2024-11-25T12:58:47,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742099_1275 (size=12731) 2024-11-25T12:58:47,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742100_1276 (size=12731) 2024-11-25T12:58:47,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:47,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1732539587107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:47,115 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:47,115 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:47,115 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:47,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:47,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:47,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52942 deadline: 1732539587112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:47,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:47,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:47,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:47,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:47,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:47,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:47,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52996 deadline: 1732539587117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:47,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:47,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:47,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:47,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-25T12:58:47,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:47,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1732539587223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:47,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:47,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:47,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:47,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:47,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52942 deadline: 1732539587228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:47,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:47,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53022 deadline: 1732539587228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:47,239 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:47,239 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:47,239 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:47,240 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:47,240 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:47,240 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:47,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:47,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52996 deadline: 1732539587240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:47,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:47,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:47,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:47,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:47,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1732539587445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:47,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:47,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:47,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:47,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:47,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52996 deadline: 1732539587450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:47,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:47,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:47,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:47,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:47,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53022 deadline: 1732539587450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:47,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:47,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:47,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:47,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:47,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52942 deadline: 1732539587450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:47,455 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:47,455 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:47,455 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:47,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-25T12:58:47,499 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/b27934e82ff5491e82f87e2d924b0245 2024-11-25T12:58:47,543 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/1c2f52fa996e47d2b8db5cddfb8427d3 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/1c2f52fa996e47d2b8db5cddfb8427d3 2024-11-25T12:58:47,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/f9b0fa0f43204160bc5c8ad1e4bb43f6 is 50, key is test_row_0/B:col10/1732539526420/Put/seqid=0 2024-11-25T12:58:47,553 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/e40085c8dbec47f6912b3c4d47f97d2c as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/e40085c8dbec47f6912b3c4d47f97d2c 2024-11-25T12:58:47,564 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21ea60aa3027cec780fb8ce3ec8872d3/A of 21ea60aa3027cec780fb8ce3ec8872d3 into 1c2f52fa996e47d2b8db5cddfb8427d3(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:58:47,564 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21ea60aa3027cec780fb8ce3ec8872d3: 2024-11-25T12:58:47,564 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., storeName=21ea60aa3027cec780fb8ce3ec8872d3/A, priority=13, startTime=1732539526997; duration=0sec 2024-11-25T12:58:47,564 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:58:47,564 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21ea60aa3027cec780fb8ce3ec8872d3:A 2024-11-25T12:58:47,564 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:58:47,581 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21ea60aa3027cec780fb8ce3ec8872d3/B of 21ea60aa3027cec780fb8ce3ec8872d3 into e40085c8dbec47f6912b3c4d47f97d2c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:58:47,581 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21ea60aa3027cec780fb8ce3ec8872d3: 2024-11-25T12:58:47,581 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., storeName=21ea60aa3027cec780fb8ce3ec8872d3/B, priority=13, startTime=1732539526997; duration=0sec 2024-11-25T12:58:47,582 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:58:47,582 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21ea60aa3027cec780fb8ce3ec8872d3:B 2024-11-25T12:58:47,582 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:58:47,582 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1541): 21ea60aa3027cec780fb8ce3ec8872d3/C is initiating minor compaction (all files) 2024-11-25T12:58:47,583 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21ea60aa3027cec780fb8ce3ec8872d3/C in TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:47,583 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/126258b976a147e8b9c8fdcf5281bab5, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/b5477000f70c477fbb4283c04ac1e3e5, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/022755c065b943ef91f833a5c86fd5a9] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp, totalSize=36.1 K 2024-11-25T12:58:47,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742101_1277 (size=12301) 2024-11-25T12:58:47,584 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 126258b976a147e8b9c8fdcf5281bab5, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732539524143 2024-11-25T12:58:47,584 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/f9b0fa0f43204160bc5c8ad1e4bb43f6 2024-11-25T12:58:47,586 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5477000f70c477fbb4283c04ac1e3e5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1732539524488 2024-11-25T12:58:47,587 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 022755c065b943ef91f833a5c86fd5a9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732539525213 2024-11-25T12:58:47,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/357988c2337e450da1d1a763bd4fad0a is 50, key is test_row_0/C:col10/1732539526420/Put/seqid=0 2024-11-25T12:58:47,624 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21ea60aa3027cec780fb8ce3ec8872d3#C#compaction#233 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:58:47,625 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/0bc07e18a67644bf83cd6eeb3674063c is 50, key is test_row_0/C:col10/1732539526375/Put/seqid=0 2024-11-25T12:58:47,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:47,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53014 deadline: 1732539587622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:47,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:47,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:47,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:47,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742102_1278 (size=12301) 2024-11-25T12:58:47,691 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/357988c2337e450da1d1a763bd4fad0a 2024-11-25T12:58:47,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742103_1279 (size=12731) 2024-11-25T12:58:47,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/b27934e82ff5491e82f87e2d924b0245 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/b27934e82ff5491e82f87e2d924b0245 2024-11-25T12:58:47,709 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/0bc07e18a67644bf83cd6eeb3674063c as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/0bc07e18a67644bf83cd6eeb3674063c 2024-11-25T12:58:47,714 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/b27934e82ff5491e82f87e2d924b0245, entries=150, sequenceid=270, filesize=12.0 K 2024-11-25T12:58:47,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/f9b0fa0f43204160bc5c8ad1e4bb43f6 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/f9b0fa0f43204160bc5c8ad1e4bb43f6 2024-11-25T12:58:47,732 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21ea60aa3027cec780fb8ce3ec8872d3/C of 21ea60aa3027cec780fb8ce3ec8872d3 into 0bc07e18a67644bf83cd6eeb3674063c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:58:47,732 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21ea60aa3027cec780fb8ce3ec8872d3: 2024-11-25T12:58:47,732 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., storeName=21ea60aa3027cec780fb8ce3ec8872d3/C, priority=13, startTime=1732539526997; duration=0sec 2024-11-25T12:58:47,732 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:58:47,732 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21ea60aa3027cec780fb8ce3ec8872d3:C 2024-11-25T12:58:47,735 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/f9b0fa0f43204160bc5c8ad1e4bb43f6, entries=150, sequenceid=270, filesize=12.0 K 2024-11-25T12:58:47,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/357988c2337e450da1d1a763bd4fad0a as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/357988c2337e450da1d1a763bd4fad0a 2024-11-25T12:58:47,744 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/357988c2337e450da1d1a763bd4fad0a, entries=150, sequenceid=270, filesize=12.0 K 2024-11-25T12:58:47,751 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 21ea60aa3027cec780fb8ce3ec8872d3 in 745ms, sequenceid=270, compaction requested=false 2024-11-25T12:58:47,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 21ea60aa3027cec780fb8ce3ec8872d3: 2024-11-25T12:58:47,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:47,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-25T12:58:47,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-25T12:58:47,756 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-25T12:58:47,756 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 901 msec 2024-11-25T12:58:47,763 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 910 msec 2024-11-25T12:58:47,771 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:58:47,771 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 21ea60aa3027cec780fb8ce3ec8872d3 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-25T12:58:47,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 21ea60aa3027cec780fb8ce3ec8872d3 2024-11-25T12:58:47,783 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/a596aa19e80045c2b2a63c47c6652496 is 50, key is test_row_0/A:col10/1732539527767/Put/seqid=0 2024-11-25T12:58:47,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742104_1280 (size=14741) 2024-11-25T12:58:47,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:47,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52942 deadline: 1732539587792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:47,833 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:47,833 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:47,833 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:47,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:47,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53022 deadline: 1732539587833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:47,837 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:47,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:47,837 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:47,837 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:47,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52996 deadline: 1732539587833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:47,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:47,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:47,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:47,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:47,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1732539587833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:47,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:47,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:47,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:47,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:47,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52942 deadline: 1732539587940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:47,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:47,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52996 deadline: 1732539587940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:47,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:47,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:47,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:47,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:47,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53022 deadline: 1732539587940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:47,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:47,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1732539587940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:47,944 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:47,944 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:47,944 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:47,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:47,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:47,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:47,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:47,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:47,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:47,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-25T12:58:47,990 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:58:47,990 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:58:47,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-25T12:58:47,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-25T12:58:47,993 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:58:47,994 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:58:47,994 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:58:48,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-25T12:58:48,145 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-25T12:58:48,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:48,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. as already flushing 2024-11-25T12:58:48,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:48,146 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:48,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:48,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:48,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:48,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52942 deadline: 1732539588151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:48,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:48,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52996 deadline: 1732539588151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:48,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:48,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53022 deadline: 1732539588151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:48,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:48,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:48,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:48,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1732539588151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:48,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:48,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:48,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:48,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:48,154 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:48,154 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:48,154 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:48,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:48,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:48,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:48,210 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/a596aa19e80045c2b2a63c47c6652496 2024-11-25T12:58:48,246 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/4eea3421ec53485aa60aed231cc906e8 is 50, key is test_row_0/B:col10/1732539527767/Put/seqid=0 2024-11-25T12:58:48,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742105_1281 (size=12301) 2024-11-25T12:58:48,299 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-25T12:58:48,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:48,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. as already flushing 2024-11-25T12:58:48,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:48,299 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:48,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:48,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:48,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-25T12:58:48,457 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-25T12:58:48,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:48,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. as already flushing 2024-11-25T12:58:48,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:48,458 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:48,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:48,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:48,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:48,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52996 deadline: 1732539588460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:48,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:48,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1732539588460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:48,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:48,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:48,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:48,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:48,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:48,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:48,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:48,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52942 deadline: 1732539588460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:48,466 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:48,466 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:48,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:48,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:48,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53022 deadline: 1732539588461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:48,467 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:48,467 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:48,467 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:48,616 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-25T12:58:48,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:48,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. as already flushing 2024-11-25T12:58:48,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:48,616 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:48,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:48,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:48,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-25T12:58:48,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:48,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53014 deadline: 1732539588640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:48,644 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 1ff8009e463e,35905,1732539449246 for 'test_row_1' in 21ea60aa3027cec780fb8ce3ec8872d3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T12:58:48,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:48,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:48,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:48,697 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/4eea3421ec53485aa60aed231cc906e8 2024-11-25T12:58:48,709 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/3d677843a89a4673a9754e6d112e795a is 50, key is test_row_0/C:col10/1732539527767/Put/seqid=0 2024-11-25T12:58:48,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742106_1282 (size=12301) 2024-11-25T12:58:48,723 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/3d677843a89a4673a9754e6d112e795a 2024-11-25T12:58:48,730 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/a596aa19e80045c2b2a63c47c6652496 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/a596aa19e80045c2b2a63c47c6652496 2024-11-25T12:58:48,758 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/a596aa19e80045c2b2a63c47c6652496, entries=200, sequenceid=296, filesize=14.4 K 2024-11-25T12:58:48,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/4eea3421ec53485aa60aed231cc906e8 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/4eea3421ec53485aa60aed231cc906e8 2024-11-25T12:58:48,770 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-25T12:58:48,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:48,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. as already flushing 2024-11-25T12:58:48,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:48,770 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:48,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:48,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:48,777 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/4eea3421ec53485aa60aed231cc906e8, entries=150, sequenceid=296, filesize=12.0 K 2024-11-25T12:58:48,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/3d677843a89a4673a9754e6d112e795a as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/3d677843a89a4673a9754e6d112e795a 2024-11-25T12:58:48,785 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/3d677843a89a4673a9754e6d112e795a, entries=150, sequenceid=296, filesize=12.0 K 2024-11-25T12:58:48,796 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 21ea60aa3027cec780fb8ce3ec8872d3 in 1025ms, sequenceid=296, compaction requested=true 2024-11-25T12:58:48,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 21ea60aa3027cec780fb8ce3ec8872d3: 2024-11-25T12:58:48,796 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:58:48,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21ea60aa3027cec780fb8ce3ec8872d3:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:58:48,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:58:48,797 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:58:48,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21ea60aa3027cec780fb8ce3ec8872d3:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:58:48,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:58:48,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21ea60aa3027cec780fb8ce3ec8872d3:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:58:48,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:58:48,797 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39773 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:58:48,798 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1541): 21ea60aa3027cec780fb8ce3ec8872d3/A is initiating minor compaction (all files) 2024-11-25T12:58:48,798 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21ea60aa3027cec780fb8ce3ec8872d3/A in TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:48,798 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/1c2f52fa996e47d2b8db5cddfb8427d3, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/b27934e82ff5491e82f87e2d924b0245, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/a596aa19e80045c2b2a63c47c6652496] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp, totalSize=38.8 K 2024-11-25T12:58:48,798 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c2f52fa996e47d2b8db5cddfb8427d3, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732539525213 2024-11-25T12:58:48,799 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting b27934e82ff5491e82f87e2d924b0245, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1732539526400 2024-11-25T12:58:48,799 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:58:48,799 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): 21ea60aa3027cec780fb8ce3ec8872d3/B is initiating minor compaction (all files) 2024-11-25T12:58:48,799 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21ea60aa3027cec780fb8ce3ec8872d3/B in TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:48,799 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/e40085c8dbec47f6912b3c4d47f97d2c, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/f9b0fa0f43204160bc5c8ad1e4bb43f6, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/4eea3421ec53485aa60aed231cc906e8] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp, totalSize=36.5 K 2024-11-25T12:58:48,799 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting a596aa19e80045c2b2a63c47c6652496, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1732539527086 2024-11-25T12:58:48,800 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting e40085c8dbec47f6912b3c4d47f97d2c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732539525213 2024-11-25T12:58:48,800 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting f9b0fa0f43204160bc5c8ad1e4bb43f6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1732539526400 2024-11-25T12:58:48,801 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 4eea3421ec53485aa60aed231cc906e8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1732539527086 2024-11-25T12:58:48,833 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21ea60aa3027cec780fb8ce3ec8872d3#A#compaction#237 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:58:48,834 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/2760aadc3c6b45d4a231c9dd84b0bfcc is 50, key is test_row_0/A:col10/1732539527767/Put/seqid=0 2024-11-25T12:58:48,854 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21ea60aa3027cec780fb8ce3ec8872d3#B#compaction#238 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:58:48,856 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/c4967385e672492c942ac047149c6e87 is 50, key is test_row_0/B:col10/1732539527767/Put/seqid=0 2024-11-25T12:58:48,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742107_1283 (size=12983) 2024-11-25T12:58:48,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742108_1284 (size=12983) 2024-11-25T12:58:48,925 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-25T12:58:48,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:48,926 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 21ea60aa3027cec780fb8ce3ec8872d3 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-25T12:58:48,926 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/2760aadc3c6b45d4a231c9dd84b0bfcc as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/2760aadc3c6b45d4a231c9dd84b0bfcc 2024-11-25T12:58:48,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/41f091d4fdb44dbea1b6f6a1b849c500 is 50, key is test_row_0/A:col10/1732539527782/Put/seqid=0 2024-11-25T12:58:48,940 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21ea60aa3027cec780fb8ce3ec8872d3/A of 21ea60aa3027cec780fb8ce3ec8872d3 into 2760aadc3c6b45d4a231c9dd84b0bfcc(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:58:48,940 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21ea60aa3027cec780fb8ce3ec8872d3: 2024-11-25T12:58:48,940 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., storeName=21ea60aa3027cec780fb8ce3ec8872d3/A, priority=13, startTime=1732539528796; duration=0sec 2024-11-25T12:58:48,940 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:58:48,940 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21ea60aa3027cec780fb8ce3ec8872d3:A 2024-11-25T12:58:48,940 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:58:48,943 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:58:48,944 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1541): 21ea60aa3027cec780fb8ce3ec8872d3/C is initiating minor compaction (all files) 2024-11-25T12:58:48,944 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21ea60aa3027cec780fb8ce3ec8872d3/C in TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:48,944 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/0bc07e18a67644bf83cd6eeb3674063c, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/357988c2337e450da1d1a763bd4fad0a, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/3d677843a89a4673a9754e6d112e795a] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp, totalSize=36.5 K 2024-11-25T12:58:48,947 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0bc07e18a67644bf83cd6eeb3674063c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732539525213 2024-11-25T12:58:48,947 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 357988c2337e450da1d1a763bd4fad0a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1732539526400 2024-11-25T12:58:48,951 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d677843a89a4673a9754e6d112e795a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1732539527086 2024-11-25T12:58:48,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742109_1285 (size=12301) 2024-11-25T12:58:48,967 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/41f091d4fdb44dbea1b6f6a1b849c500 2024-11-25T12:58:48,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. as already flushing 2024-11-25T12:58:48,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 21ea60aa3027cec780fb8ce3ec8872d3 2024-11-25T12:58:48,980 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21ea60aa3027cec780fb8ce3ec8872d3#C#compaction#240 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:58:48,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/17291e965c08450f977639620542f89c is 50, key is test_row_0/B:col10/1732539527782/Put/seqid=0 2024-11-25T12:58:48,980 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/d11ea32e39024acc8365a535d08a0f6e is 50, key is test_row_0/C:col10/1732539527767/Put/seqid=0 2024-11-25T12:58:49,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742110_1286 (size=12301) 2024-11-25T12:58:49,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742111_1287 (size=12983) 2024-11-25T12:58:49,092 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/17291e965c08450f977639620542f89c 2024-11-25T12:58:49,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:49,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1732539589088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:49,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:49,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:49,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:49,114 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/d11ea32e39024acc8365a535d08a0f6e as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/d11ea32e39024acc8365a535d08a0f6e 2024-11-25T12:58:49,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:49,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52942 deadline: 1732539589095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:49,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:49,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53022 deadline: 1732539589106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:49,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:49,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:49,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:49,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52996 deadline: 1732539589109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:49,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:49,122 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:49,122 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:49,122 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:49,122 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:49,122 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:49,122 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:49,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-25T12:58:49,132 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21ea60aa3027cec780fb8ce3ec8872d3/C of 21ea60aa3027cec780fb8ce3ec8872d3 into d11ea32e39024acc8365a535d08a0f6e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:58:49,132 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21ea60aa3027cec780fb8ce3ec8872d3: 2024-11-25T12:58:49,132 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., storeName=21ea60aa3027cec780fb8ce3ec8872d3/C, priority=13, startTime=1732539528797; duration=0sec 2024-11-25T12:58:49,132 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:58:49,132 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21ea60aa3027cec780fb8ce3ec8872d3:C 2024-11-25T12:58:49,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/994102e79f8f4f97a2807a588bd65a56 is 50, key is test_row_0/C:col10/1732539527782/Put/seqid=0 2024-11-25T12:58:49,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742112_1288 (size=12301) 2024-11-25T12:58:49,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:49,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1732539589221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:49,223 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:49,223 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:49,223 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:49,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:49,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53022 deadline: 1732539589230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:49,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:49,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52996 deadline: 1732539589230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:49,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:49,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:49,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:49,231 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:49,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:49,231 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:49,231 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:49,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52942 deadline: 1732539589230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:49,231 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:49,231 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:49,232 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:49,326 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/c4967385e672492c942ac047149c6e87 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/c4967385e672492c942ac047149c6e87 2024-11-25T12:58:49,330 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21ea60aa3027cec780fb8ce3ec8872d3/B of 21ea60aa3027cec780fb8ce3ec8872d3 into c4967385e672492c942ac047149c6e87(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:58:49,330 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21ea60aa3027cec780fb8ce3ec8872d3: 2024-11-25T12:58:49,330 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., storeName=21ea60aa3027cec780fb8ce3ec8872d3/B, priority=13, startTime=1732539528797; duration=0sec 2024-11-25T12:58:49,330 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:58:49,331 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21ea60aa3027cec780fb8ce3ec8872d3:B 2024-11-25T12:58:49,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:49,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1732539589430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:49,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:49,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:49,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:49,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:49,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53022 deadline: 1732539589439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:49,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:49,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:49,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:49,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:49,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52942 deadline: 1732539589440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:49,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:49,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52996 deadline: 1732539589440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:49,441 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:49,441 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:49,441 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:49,441 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:49,441 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:49,441 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:49,540 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/994102e79f8f4f97a2807a588bd65a56 2024-11-25T12:58:49,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/41f091d4fdb44dbea1b6f6a1b849c500 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/41f091d4fdb44dbea1b6f6a1b849c500 2024-11-25T12:58:49,550 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/41f091d4fdb44dbea1b6f6a1b849c500, entries=150, sequenceid=309, filesize=12.0 K 2024-11-25T12:58:49,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/17291e965c08450f977639620542f89c as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/17291e965c08450f977639620542f89c 2024-11-25T12:58:49,560 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/17291e965c08450f977639620542f89c, entries=150, sequenceid=309, filesize=12.0 K 2024-11-25T12:58:49,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/994102e79f8f4f97a2807a588bd65a56 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/994102e79f8f4f97a2807a588bd65a56 2024-11-25T12:58:49,567 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/994102e79f8f4f97a2807a588bd65a56, entries=150, sequenceid=309, filesize=12.0 K 2024-11-25T12:58:49,568 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 21ea60aa3027cec780fb8ce3ec8872d3 in 642ms, sequenceid=309, compaction requested=false 2024-11-25T12:58:49,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 21ea60aa3027cec780fb8ce3ec8872d3: 2024-11-25T12:58:49,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:49,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-25T12:58:49,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-25T12:58:49,572 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-25T12:58:49,572 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5760 sec 2024-11-25T12:58:49,575 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.5830 sec 2024-11-25T12:58:49,743 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:58:49,743 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 21ea60aa3027cec780fb8ce3ec8872d3 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-25T12:58:49,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 21ea60aa3027cec780fb8ce3ec8872d3 2024-11-25T12:58:49,757 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/5bb8e5feb31d4fba867e8b3cd84662f1 is 50, key is test_row_0/A:col10/1732539529089/Put/seqid=0 2024-11-25T12:58:49,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742113_1289 (size=14741) 2024-11-25T12:58:49,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:49,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1732539589796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:49,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:49,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:49,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:49,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:49,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52942 deadline: 1732539589799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:49,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:49,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:49,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:49,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:49,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53022 deadline: 1732539589801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:49,807 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:49,807 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:49,807 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:49,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:49,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52996 deadline: 1732539589805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:49,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:49,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:49,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:49,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:49,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:49,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53022 deadline: 1732539589910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:49,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1732539589910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:49,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:49,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52942 deadline: 1732539589910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:49,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:49,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:49,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:49,919 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:49,919 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:49,919 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:49,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:49,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:49,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:49,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:49,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52996 deadline: 1732539589920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:49,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:49,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:49,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:50,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:50,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53022 deadline: 1732539590120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:50,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:50,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:50,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:50,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:50,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52942 deadline: 1732539590130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:50,135 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:50,135 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:50,135 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:50,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:50,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1732539590130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:50,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:50,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:50,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:50,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:50,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52996 deadline: 1732539590130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:50,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:50,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:50,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:50,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-25T12:58:50,147 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:58:50,148 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:58:50,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-25T12:58:50,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-25T12:58:50,152 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:58:50,153 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:58:50,153 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:58:50,178 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/5bb8e5feb31d4fba867e8b3cd84662f1 2024-11-25T12:58:50,196 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/bea503c95cca49429b488345d735cea2 is 50, key is test_row_0/B:col10/1732539529089/Put/seqid=0 2024-11-25T12:58:50,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742114_1290 (size=12301) 2024-11-25T12:58:50,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-25T12:58:50,307 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-25T12:58:50,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:50,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. as already flushing 2024-11-25T12:58:50,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:50,307 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:50,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:50,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:50,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:50,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53022 deadline: 1732539590430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:50,438 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:50,438 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:50,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:50,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:50,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52996 deadline: 1732539590443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:50,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:50,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52942 deadline: 1732539590443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:50,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:50,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1732539590443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:50,448 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:50,448 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:50,448 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:50,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:50,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:50,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:50,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:50,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:50,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:50,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-25T12:58:50,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:50,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. as already flushing 2024-11-25T12:58:50,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:50,471 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:50,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:50,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-25T12:58:50,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:50,625 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-25T12:58:50,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:50,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. as already flushing 2024-11-25T12:58:50,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:50,625 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:50,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:50,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:50,656 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/bea503c95cca49429b488345d735cea2 2024-11-25T12:58:50,674 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/f1ec1940cbfa43dbba0cf36e5c381950 is 50, key is test_row_0/C:col10/1732539529089/Put/seqid=0 2024-11-25T12:58:50,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:50,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53014 deadline: 1732539590670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:50,675 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 1ff8009e463e,35905,1732539449246 for 'test_row_1' in 21ea60aa3027cec780fb8ce3ec8872d3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4219 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T12:58:50,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:50,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:50,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:50,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742115_1291 (size=12301) 2024-11-25T12:58:50,777 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-25T12:58:50,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:50,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. as already flushing 2024-11-25T12:58:50,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:50,778 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:50,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:50,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:50,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-25T12:58:50,948 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-25T12:58:50,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:50,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. as already flushing 2024-11-25T12:58:50,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:50,950 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:50,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:50,952 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:50,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53022 deadline: 1732539590950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:50,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:50,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:50,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:50,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:50,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:50,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52996 deadline: 1732539590960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:50,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:50,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:50,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:50,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52942 deadline: 1732539590960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:50,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:50,964 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:50,964 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:50,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:50,964 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:50,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1732539590960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:50,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:50,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:50,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:51,115 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-25T12:58:51,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:51,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. as already flushing 2024-11-25T12:58:51,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:51,116 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:51,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:51,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:51,122 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/f1ec1940cbfa43dbba0cf36e5c381950 2024-11-25T12:58:51,130 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/5bb8e5feb31d4fba867e8b3cd84662f1 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/5bb8e5feb31d4fba867e8b3cd84662f1 2024-11-25T12:58:51,141 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/5bb8e5feb31d4fba867e8b3cd84662f1, entries=200, sequenceid=336, filesize=14.4 K 2024-11-25T12:58:51,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/bea503c95cca49429b488345d735cea2 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/bea503c95cca49429b488345d735cea2 2024-11-25T12:58:51,176 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/bea503c95cca49429b488345d735cea2, entries=150, sequenceid=336, filesize=12.0 K 2024-11-25T12:58:51,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/f1ec1940cbfa43dbba0cf36e5c381950 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/f1ec1940cbfa43dbba0cf36e5c381950 2024-11-25T12:58:51,194 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/f1ec1940cbfa43dbba0cf36e5c381950, entries=150, sequenceid=336, filesize=12.0 K 2024-11-25T12:58:51,197 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=73.80 KB/75570 for 21ea60aa3027cec780fb8ce3ec8872d3 in 1454ms, sequenceid=336, compaction requested=true 2024-11-25T12:58:51,197 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 21ea60aa3027cec780fb8ce3ec8872d3: 2024-11-25T12:58:51,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21ea60aa3027cec780fb8ce3ec8872d3:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:58:51,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:58:51,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21ea60aa3027cec780fb8ce3ec8872d3:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:58:51,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-25T12:58:51,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21ea60aa3027cec780fb8ce3ec8872d3:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:58:51,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-25T12:58:51,197 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:58:51,197 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:58:51,199 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:58:51,199 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1541): 21ea60aa3027cec780fb8ce3ec8872d3/A is initiating minor compaction (all files) 2024-11-25T12:58:51,200 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21ea60aa3027cec780fb8ce3ec8872d3/A in TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:51,200 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/2760aadc3c6b45d4a231c9dd84b0bfcc, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/41f091d4fdb44dbea1b6f6a1b849c500, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/5bb8e5feb31d4fba867e8b3cd84662f1] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp, totalSize=39.1 K 2024-11-25T12:58:51,200 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:58:51,200 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): 21ea60aa3027cec780fb8ce3ec8872d3/C is initiating minor compaction (all files) 2024-11-25T12:58:51,200 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21ea60aa3027cec780fb8ce3ec8872d3/C in TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:51,200 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/d11ea32e39024acc8365a535d08a0f6e, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/994102e79f8f4f97a2807a588bd65a56, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/f1ec1940cbfa43dbba0cf36e5c381950] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp, totalSize=36.7 K 2024-11-25T12:58:51,201 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting d11ea32e39024acc8365a535d08a0f6e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1732539527086 2024-11-25T12:58:51,201 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2760aadc3c6b45d4a231c9dd84b0bfcc, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1732539527086 2024-11-25T12:58:51,201 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 994102e79f8f4f97a2807a588bd65a56, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732539527782 2024-11-25T12:58:51,201 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41f091d4fdb44dbea1b6f6a1b849c500, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732539527782 2024-11-25T12:58:51,202 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting f1ec1940cbfa43dbba0cf36e5c381950, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1732539529033 2024-11-25T12:58:51,202 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5bb8e5feb31d4fba867e8b3cd84662f1, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1732539529033 2024-11-25T12:58:51,237 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21ea60aa3027cec780fb8ce3ec8872d3#C#compaction#246 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:58:51,238 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/5912802b9067463da6d2fe35d7451136 is 50, key is test_row_0/C:col10/1732539529089/Put/seqid=0 2024-11-25T12:58:51,257 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21ea60aa3027cec780fb8ce3ec8872d3#A#compaction#247 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:58:51,258 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/aa44964cc6564e44bd6d04e082718c01 is 50, key is test_row_0/A:col10/1732539529089/Put/seqid=0 2024-11-25T12:58:51,275 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-25T12:58:51,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:51,276 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 21ea60aa3027cec780fb8ce3ec8872d3 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-25T12:58:51,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-25T12:58:51,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742116_1292 (size=13085) 2024-11-25T12:58:51,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742117_1293 (size=13085) 2024-11-25T12:58:51,335 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/aa44964cc6564e44bd6d04e082718c01 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/aa44964cc6564e44bd6d04e082718c01 2024-11-25T12:58:51,341 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21ea60aa3027cec780fb8ce3ec8872d3/A of 21ea60aa3027cec780fb8ce3ec8872d3 into aa44964cc6564e44bd6d04e082718c01(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:58:51,341 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21ea60aa3027cec780fb8ce3ec8872d3: 2024-11-25T12:58:51,341 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., storeName=21ea60aa3027cec780fb8ce3ec8872d3/A, priority=13, startTime=1732539531197; duration=0sec 2024-11-25T12:58:51,341 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:58:51,341 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21ea60aa3027cec780fb8ce3ec8872d3:A 2024-11-25T12:58:51,341 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:58:51,342 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:58:51,342 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1541): 21ea60aa3027cec780fb8ce3ec8872d3/B is initiating minor compaction (all files) 2024-11-25T12:58:51,342 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21ea60aa3027cec780fb8ce3ec8872d3/B in TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:51,342 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/c4967385e672492c942ac047149c6e87, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/17291e965c08450f977639620542f89c, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/bea503c95cca49429b488345d735cea2] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp, totalSize=36.7 K 2024-11-25T12:58:51,342 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4967385e672492c942ac047149c6e87, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1732539527086 2024-11-25T12:58:51,343 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17291e965c08450f977639620542f89c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732539527782 2024-11-25T12:58:51,343 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting bea503c95cca49429b488345d735cea2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1732539529033 2024-11-25T12:58:51,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/51773ac09fcb4bce8fb3eaf6426d75ca is 50, key is test_row_0/A:col10/1732539529796/Put/seqid=0 2024-11-25T12:58:51,360 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21ea60aa3027cec780fb8ce3ec8872d3#B#compaction#249 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:58:51,361 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/34f0dee6793c4d74bf8248e0302cc04a is 50, key is test_row_0/B:col10/1732539529089/Put/seqid=0 2024-11-25T12:58:51,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742118_1294 (size=12301) 2024-11-25T12:58:51,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742119_1295 (size=13085) 2024-11-25T12:58:51,732 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/5912802b9067463da6d2fe35d7451136 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/5912802b9067463da6d2fe35d7451136 2024-11-25T12:58:51,742 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21ea60aa3027cec780fb8ce3ec8872d3/C of 21ea60aa3027cec780fb8ce3ec8872d3 into 5912802b9067463da6d2fe35d7451136(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:58:51,742 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21ea60aa3027cec780fb8ce3ec8872d3: 2024-11-25T12:58:51,742 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., storeName=21ea60aa3027cec780fb8ce3ec8872d3/C, priority=13, startTime=1732539531197; duration=0sec 2024-11-25T12:58:51,742 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:58:51,742 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21ea60aa3027cec780fb8ce3ec8872d3:C 2024-11-25T12:58:51,793 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/51773ac09fcb4bce8fb3eaf6426d75ca 2024-11-25T12:58:51,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/a95d5c54af604865891bd5a117b7e6e0 is 50, key is test_row_0/B:col10/1732539529796/Put/seqid=0 2024-11-25T12:58:51,841 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/34f0dee6793c4d74bf8248e0302cc04a as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/34f0dee6793c4d74bf8248e0302cc04a 2024-11-25T12:58:51,851 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21ea60aa3027cec780fb8ce3ec8872d3/B of 21ea60aa3027cec780fb8ce3ec8872d3 into 34f0dee6793c4d74bf8248e0302cc04a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:58:51,851 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21ea60aa3027cec780fb8ce3ec8872d3: 2024-11-25T12:58:51,851 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., storeName=21ea60aa3027cec780fb8ce3ec8872d3/B, priority=13, startTime=1732539531197; duration=0sec 2024-11-25T12:58:51,854 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:58:51,854 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21ea60aa3027cec780fb8ce3ec8872d3:B 2024-11-25T12:58:51,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742120_1296 (size=12301) 2024-11-25T12:58:51,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 21ea60aa3027cec780fb8ce3ec8872d3 2024-11-25T12:58:51,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. as already flushing 2024-11-25T12:58:51,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:51,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52942 deadline: 1732539591989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:51,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:51,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:51,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:51,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:51,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1732539591990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:51,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:51,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:51,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:51,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:51,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53022 deadline: 1732539591991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:51,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:51,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:51,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:51,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:51,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52996 deadline: 1732539591994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:51,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:51,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:51,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:52,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:52,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52942 deadline: 1732539592100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:52,102 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:52,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:52,102 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:52,102 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:52,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53022 deadline: 1732539592100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:52,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:52,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1732539592100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:52,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:52,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52996 deadline: 1732539592100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:52,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:52,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:52,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:52,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:52,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:52,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:52,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:52,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:52,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:52,268 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/a95d5c54af604865891bd5a117b7e6e0 2024-11-25T12:58:52,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/af49e2d083bf4a3c96bd2425db26a8a6 is 50, key is test_row_0/C:col10/1732539529796/Put/seqid=0 2024-11-25T12:58:52,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-25T12:58:52,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742121_1297 (size=12301) 2024-11-25T12:58:52,302 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/af49e2d083bf4a3c96bd2425db26a8a6 2024-11-25T12:58:52,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:52,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53022 deadline: 1732539592310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:52,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:52,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:52,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52996 deadline: 1732539592310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:52,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:52,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:52,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:52,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:52,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:52,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:52,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52942 deadline: 1732539592311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:52,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:52,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1732539592311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:52,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:52,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:52,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:52,315 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:52,315 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:52,315 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:52,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/51773ac09fcb4bce8fb3eaf6426d75ca as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/51773ac09fcb4bce8fb3eaf6426d75ca 2024-11-25T12:58:52,334 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/51773ac09fcb4bce8fb3eaf6426d75ca, entries=150, sequenceid=350, filesize=12.0 K 2024-11-25T12:58:52,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/a95d5c54af604865891bd5a117b7e6e0 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/a95d5c54af604865891bd5a117b7e6e0 2024-11-25T12:58:52,341 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/a95d5c54af604865891bd5a117b7e6e0, entries=150, sequenceid=350, filesize=12.0 K 2024-11-25T12:58:52,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/af49e2d083bf4a3c96bd2425db26a8a6 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/af49e2d083bf4a3c96bd2425db26a8a6 2024-11-25T12:58:52,380 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/af49e2d083bf4a3c96bd2425db26a8a6, entries=150, sequenceid=350, filesize=12.0 K 2024-11-25T12:58:52,388 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 21ea60aa3027cec780fb8ce3ec8872d3 in 1113ms, sequenceid=350, compaction requested=false 2024-11-25T12:58:52,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 21ea60aa3027cec780fb8ce3ec8872d3: 2024-11-25T12:58:52,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:52,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-25T12:58:52,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-25T12:58:52,393 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-25T12:58:52,393 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2370 sec 2024-11-25T12:58:52,396 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 2.2450 sec 2024-11-25T12:58:52,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 21ea60aa3027cec780fb8ce3ec8872d3 2024-11-25T12:58:52,625 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:58:52,625 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 21ea60aa3027cec780fb8ce3ec8872d3 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-25T12:58:52,639 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/bb9f5828fb984363bb751a603a5c4bcb is 50, key is test_row_0/A:col10/1732539531990/Put/seqid=0 2024-11-25T12:58:52,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:52,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:52,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53022 deadline: 1732539592644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:52,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52942 deadline: 1732539592643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:52,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:52,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52996 deadline: 1732539592645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:52,648 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:52,648 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:52,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:52,648 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:52,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1732539592645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:52,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:52,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:52,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:52,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:52,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:52,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:52,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:52,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:52,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:52,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742122_1298 (size=12301) 2024-11-25T12:58:52,654 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/bb9f5828fb984363bb751a603a5c4bcb 2024-11-25T12:58:52,664 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/26cc934c03204482b91e720bf60c5408 is 50, key is test_row_0/B:col10/1732539531990/Put/seqid=0 2024-11-25T12:58:52,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742123_1299 (size=12301) 2024-11-25T12:58:52,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:52,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52942 deadline: 1732539592760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:52,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:52,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53022 deadline: 1732539592760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:52,765 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:52,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:52,765 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:52,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1732539592760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:52,765 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:52,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:52,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:52,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:52,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:52,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:52,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:52,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:52,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52996 deadline: 1732539592760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:52,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:52,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:52,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:52,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:52,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1732539592972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:52,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:52,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:52,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:52,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:52,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53022 deadline: 1732539592972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:52,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:52,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52996 deadline: 1732539592972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:52,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:52,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:52,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:52,979 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:52,979 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:52,979 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:52,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:52,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52942 deadline: 1732539592975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:52,984 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:52,984 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:52,984 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:53,119 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/26cc934c03204482b91e720bf60c5408 2024-11-25T12:58:53,131 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/36ce8dde402c4d24a0dc0ce8249cbdb6 is 50, key is test_row_0/C:col10/1732539531990/Put/seqid=0 2024-11-25T12:58:53,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742124_1300 (size=12301) 2024-11-25T12:58:53,145 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/36ce8dde402c4d24a0dc0ce8249cbdb6 2024-11-25T12:58:53,163 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/bb9f5828fb984363bb751a603a5c4bcb as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/bb9f5828fb984363bb751a603a5c4bcb 2024-11-25T12:58:53,169 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/bb9f5828fb984363bb751a603a5c4bcb, entries=150, sequenceid=376, filesize=12.0 K 2024-11-25T12:58:53,170 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/26cc934c03204482b91e720bf60c5408 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/26cc934c03204482b91e720bf60c5408 2024-11-25T12:58:53,175 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/26cc934c03204482b91e720bf60c5408, entries=150, sequenceid=376, filesize=12.0 K 2024-11-25T12:58:53,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/36ce8dde402c4d24a0dc0ce8249cbdb6 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/36ce8dde402c4d24a0dc0ce8249cbdb6 2024-11-25T12:58:53,183 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/36ce8dde402c4d24a0dc0ce8249cbdb6, entries=150, sequenceid=376, filesize=12.0 K 2024-11-25T12:58:53,185 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 21ea60aa3027cec780fb8ce3ec8872d3 in 559ms, sequenceid=376, compaction requested=true 2024-11-25T12:58:53,185 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 21ea60aa3027cec780fb8ce3ec8872d3: 2024-11-25T12:58:53,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21ea60aa3027cec780fb8ce3ec8872d3:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:58:53,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:58:53,185 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:58:53,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21ea60aa3027cec780fb8ce3ec8872d3:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:58:53,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:58:53,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21ea60aa3027cec780fb8ce3ec8872d3:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:58:53,185 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:58:53,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:58:53,186 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:58:53,186 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:58:53,186 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): 21ea60aa3027cec780fb8ce3ec8872d3/B is initiating minor compaction (all files) 2024-11-25T12:58:53,186 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1541): 21ea60aa3027cec780fb8ce3ec8872d3/A is initiating minor compaction (all files) 2024-11-25T12:58:53,186 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21ea60aa3027cec780fb8ce3ec8872d3/B in TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:53,186 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21ea60aa3027cec780fb8ce3ec8872d3/A in TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:53,186 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/aa44964cc6564e44bd6d04e082718c01, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/51773ac09fcb4bce8fb3eaf6426d75ca, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/bb9f5828fb984363bb751a603a5c4bcb] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp, totalSize=36.8 K 2024-11-25T12:58:53,186 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/34f0dee6793c4d74bf8248e0302cc04a, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/a95d5c54af604865891bd5a117b7e6e0, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/26cc934c03204482b91e720bf60c5408] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp, totalSize=36.8 K 2024-11-25T12:58:53,187 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 34f0dee6793c4d74bf8248e0302cc04a, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1732539529033 2024-11-25T12:58:53,187 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa44964cc6564e44bd6d04e082718c01, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1732539529033 2024-11-25T12:58:53,187 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting a95d5c54af604865891bd5a117b7e6e0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1732539529775 2024-11-25T12:58:53,187 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51773ac09fcb4bce8fb3eaf6426d75ca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1732539529775 2024-11-25T12:58:53,187 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 26cc934c03204482b91e720bf60c5408, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1732539531990 2024-11-25T12:58:53,188 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb9f5828fb984363bb751a603a5c4bcb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1732539531990 2024-11-25T12:58:53,201 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21ea60aa3027cec780fb8ce3ec8872d3#A#compaction#255 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:58:53,202 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/f8576c377a5446b7933127ea9c6285fb is 50, key is test_row_0/A:col10/1732539531990/Put/seqid=0 2024-11-25T12:58:53,205 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21ea60aa3027cec780fb8ce3ec8872d3#B#compaction#256 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:58:53,206 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/7568500f045f4ab4af9ce2fa4aeac049 is 50, key is test_row_0/B:col10/1732539531990/Put/seqid=0 2024-11-25T12:58:53,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742126_1302 (size=13187) 2024-11-25T12:58:53,239 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/7568500f045f4ab4af9ce2fa4aeac049 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/7568500f045f4ab4af9ce2fa4aeac049 2024-11-25T12:58:53,248 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21ea60aa3027cec780fb8ce3ec8872d3/B of 21ea60aa3027cec780fb8ce3ec8872d3 into 7568500f045f4ab4af9ce2fa4aeac049(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:58:53,248 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21ea60aa3027cec780fb8ce3ec8872d3: 2024-11-25T12:58:53,248 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., storeName=21ea60aa3027cec780fb8ce3ec8872d3/B, priority=13, startTime=1732539533185; duration=0sec 2024-11-25T12:58:53,248 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:58:53,248 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21ea60aa3027cec780fb8ce3ec8872d3:B 2024-11-25T12:58:53,248 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:58:53,250 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:58:53,250 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): 21ea60aa3027cec780fb8ce3ec8872d3/C is initiating minor compaction (all files) 2024-11-25T12:58:53,250 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21ea60aa3027cec780fb8ce3ec8872d3/C in TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:53,250 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/5912802b9067463da6d2fe35d7451136, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/af49e2d083bf4a3c96bd2425db26a8a6, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/36ce8dde402c4d24a0dc0ce8249cbdb6] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp, totalSize=36.8 K 2024-11-25T12:58:53,251 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 5912802b9067463da6d2fe35d7451136, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1732539529033 2024-11-25T12:58:53,251 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting af49e2d083bf4a3c96bd2425db26a8a6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1732539529775 2024-11-25T12:58:53,252 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 36ce8dde402c4d24a0dc0ce8249cbdb6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1732539531990 2024-11-25T12:58:53,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742125_1301 (size=13187) 2024-11-25T12:58:53,263 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/f8576c377a5446b7933127ea9c6285fb as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/f8576c377a5446b7933127ea9c6285fb 2024-11-25T12:58:53,270 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21ea60aa3027cec780fb8ce3ec8872d3/A of 21ea60aa3027cec780fb8ce3ec8872d3 into f8576c377a5446b7933127ea9c6285fb(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:58:53,270 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21ea60aa3027cec780fb8ce3ec8872d3: 2024-11-25T12:58:53,270 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., storeName=21ea60aa3027cec780fb8ce3ec8872d3/A, priority=13, startTime=1732539533185; duration=0sec 2024-11-25T12:58:53,270 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:58:53,270 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21ea60aa3027cec780fb8ce3ec8872d3:A 2024-11-25T12:58:53,279 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21ea60aa3027cec780fb8ce3ec8872d3#C#compaction#257 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:58:53,280 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/7e7278833b2b408591b586a25b68e1ef is 50, key is test_row_0/C:col10/1732539531990/Put/seqid=0 2024-11-25T12:58:53,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 21ea60aa3027cec780fb8ce3ec8872d3 2024-11-25T12:58:53,284 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:58:53,284 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 21ea60aa3027cec780fb8ce3ec8872d3 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-25T12:58:53,305 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/c7b57303fd954365b8e4cf9153d77bad is 50, key is test_row_0/A:col10/1732539533282/Put/seqid=0 2024-11-25T12:58:53,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742127_1303 (size=13187) 2024-11-25T12:58:53,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742128_1304 (size=14741) 2024-11-25T12:58:53,333 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:53,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52942 deadline: 1732539593327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:53,333 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:53,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:53,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1732539593329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:53,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53022 deadline: 1732539593327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:53,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:53,334 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:53,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52996 deadline: 1732539593329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:53,334 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:53,334 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:53,334 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:53,334 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:53,334 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:53,334 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:53,334 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:53,334 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:53,334 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:53,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:53,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:53,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:53,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52942 deadline: 1732539593440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:53,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:53,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53022 deadline: 1732539593441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:53,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:53,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:53,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:53,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52996 deadline: 1732539593441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:53,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:53,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:53,446 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:53,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1732539593441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:53,446 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:53,446 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:53,446 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:53,446 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:53,446 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:53,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:53,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:53,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:53,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:53,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52942 deadline: 1732539593650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:53,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:53,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1732539593650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:53,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:53,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53022 deadline: 1732539593650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:53,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:53,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:53,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:53,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:53,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52996 deadline: 1732539593651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:53,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:53,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:53,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:53,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:53,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:53,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:53,655 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:53,655 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:53,656 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:53,708 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/c7b57303fd954365b8e4cf9153d77bad 2024-11-25T12:58:53,719 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/7e7278833b2b408591b586a25b68e1ef as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/7e7278833b2b408591b586a25b68e1ef 2024-11-25T12:58:53,727 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21ea60aa3027cec780fb8ce3ec8872d3/C of 21ea60aa3027cec780fb8ce3ec8872d3 into 7e7278833b2b408591b586a25b68e1ef(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:58:53,727 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21ea60aa3027cec780fb8ce3ec8872d3: 2024-11-25T12:58:53,727 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., storeName=21ea60aa3027cec780fb8ce3ec8872d3/C, priority=13, startTime=1732539533185; duration=0sec 2024-11-25T12:58:53,727 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:58:53,727 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21ea60aa3027cec780fb8ce3ec8872d3:C 2024-11-25T12:58:53,735 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/36dd56d3b12a4bf29c796aa0010e9719 is 50, key is test_row_0/B:col10/1732539533282/Put/seqid=0 2024-11-25T12:58:53,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742129_1305 (size=12301) 2024-11-25T12:58:53,764 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/36dd56d3b12a4bf29c796aa0010e9719 2024-11-25T12:58:53,773 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/d7fcc55c135448169c13d73e2ab21645 is 50, key is test_row_0/C:col10/1732539533282/Put/seqid=0 2024-11-25T12:58:53,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742130_1306 (size=12301) 2024-11-25T12:58:53,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:53,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53022 deadline: 1732539593960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:53,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:53,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:53,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:53,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:53,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52996 deadline: 1732539593960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:53,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:53,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:53,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:53,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:53,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1732539593960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:53,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:53,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52942 deadline: 1732539593960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:53,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:53,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:53,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:53,967 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:53,967 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:53,967 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:54,233 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/d7fcc55c135448169c13d73e2ab21645 2024-11-25T12:58:54,249 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/c7b57303fd954365b8e4cf9153d77bad as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/c7b57303fd954365b8e4cf9153d77bad 2024-11-25T12:58:54,268 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/c7b57303fd954365b8e4cf9153d77bad, entries=200, sequenceid=392, filesize=14.4 K 2024-11-25T12:58:54,269 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/36dd56d3b12a4bf29c796aa0010e9719 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/36dd56d3b12a4bf29c796aa0010e9719 2024-11-25T12:58:54,275 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/36dd56d3b12a4bf29c796aa0010e9719, entries=150, sequenceid=392, filesize=12.0 K 2024-11-25T12:58:54,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/d7fcc55c135448169c13d73e2ab21645 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/d7fcc55c135448169c13d73e2ab21645 2024-11-25T12:58:54,289 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/d7fcc55c135448169c13d73e2ab21645, entries=150, sequenceid=392, filesize=12.0 K 2024-11-25T12:58:54,290 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 21ea60aa3027cec780fb8ce3ec8872d3 in 1006ms, sequenceid=392, compaction requested=false 2024-11-25T12:58:54,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 21ea60aa3027cec780fb8ce3ec8872d3: 2024-11-25T12:58:54,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-25T12:58:54,317 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:58:54,319 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:58:54,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-25T12:58:54,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-25T12:58:54,322 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:58:54,322 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:58:54,323 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:58:54,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-25T12:58:54,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 21ea60aa3027cec780fb8ce3ec8872d3 2024-11-25T12:58:54,472 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:58:54,472 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 21ea60aa3027cec780fb8ce3ec8872d3 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-25T12:58:54,477 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/088697dd5b2f454cabff439e75bb2231 is 50, key is test_row_0/A:col10/1732539534471/Put/seqid=0 2024-11-25T12:58:54,477 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-25T12:58:54,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:54,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. as already flushing 2024-11-25T12:58:54,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:54,478 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:54,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:54,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:54,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:54,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53022 deadline: 1732539594486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:54,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:54,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1732539594486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:54,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:54,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:54,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:54,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:54,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:54,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:54,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:54,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52942 deadline: 1732539594487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:54,491 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:54,491 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:54,491 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:54,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:54,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52996 deadline: 1732539594488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:54,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742131_1307 (size=14741) 2024-11-25T12:58:54,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:54,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:54,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:54,493 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/088697dd5b2f454cabff439e75bb2231 2024-11-25T12:58:54,500 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/e177698dad8049e0a2537e081f737519 is 50, key is test_row_0/B:col10/1732539534471/Put/seqid=0 2024-11-25T12:58:54,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742132_1308 (size=12301) 2024-11-25T12:58:54,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:54,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53022 deadline: 1732539594590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:54,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:54,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:54,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:54,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:54,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1732539594601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:54,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:54,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52996 deadline: 1732539594601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:54,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:54,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:54,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:54,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:54,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:54,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:54,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:54,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52942 deadline: 1732539594601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:54,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:54,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:54,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:54,635 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-25T12:58:54,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:54,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. as already flushing 2024-11-25T12:58:54,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:54,636 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:54,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:54,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-25T12:58:54,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:54,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:54,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53014 deadline: 1732539594690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:54,693 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 1ff8009e463e,35905,1732539449246 for 'test_row_1' in 21ea60aa3027cec780fb8ce3ec8872d3 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8238 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T12:58:54,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:54,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:54,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:54,796 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-25T12:58:54,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:54,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. as already flushing 2024-11-25T12:58:54,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:54,796 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:54,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:54,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:54,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:54,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53022 deadline: 1732539594799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:54,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:54,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:54,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:54,812 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:54,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52996 deadline: 1732539594811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:54,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:54,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:54,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:54,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:54,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52942 deadline: 1732539594811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:54,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:54,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:54,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:54,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:54,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1732539594818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:54,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:54,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:54,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:54,915 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/e177698dad8049e0a2537e081f737519 2024-11-25T12:58:54,928 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/9c90a2a1795a4b9e96f0d7051d82b3b4 is 50, key is test_row_0/C:col10/1732539534471/Put/seqid=0 2024-11-25T12:58:54,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-25T12:58:54,955 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-25T12:58:54,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:54,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. as already flushing 2024-11-25T12:58:54,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:54,956 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:54,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:54,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:54,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742133_1309 (size=12301) 2024-11-25T12:58:55,083 INFO [Thread-1061 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1061. 2024-11-25T12:58:55,083 DEBUG [Thread-1061 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:58:55,083 DEBUG [Thread-1061 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:58:55,084 DEBUG [Thread-1061 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:58:55,084 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:58:55,084 INFO [Thread-1067 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1067. 2024-11-25T12:58:55,084 DEBUG [Thread-1067 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:58:55,084 DEBUG [Thread-1067 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:58:55,084 DEBUG [Thread-1067 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:58:55,084 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:58:55,084 INFO [Thread-1065 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1065. 2024-11-25T12:58:55,084 DEBUG [Thread-1065 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:58:55,085 DEBUG [Thread-1065 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:58:55,085 DEBUG [Thread-1065 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:58:55,085 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:58:55,088 INFO [Thread-1063 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1063. 2024-11-25T12:58:55,088 DEBUG [Thread-1063 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:58:55,088 DEBUG [Thread-1063 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:58:55,088 DEBUG [Thread-1063 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:58:55,088 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:58:55,088 INFO [Thread-1059 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1059. 2024-11-25T12:58:55,088 DEBUG [Thread-1059 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:58:55,088 DEBUG [Thread-1059 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:58:55,089 DEBUG [Thread-1059 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:58:55,089 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:58:55,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:55,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53022 deadline: 1732539595110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:55,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:55,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:55,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:55,111 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-25T12:58:55,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:55,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. as already flushing 2024-11-25T12:58:55,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:55,112 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:55,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:55,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:55,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:55,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:55,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52942 deadline: 1732539595120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:55,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52996 deadline: 1732539595120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:55,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:55,121 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:55,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:55,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:55,121 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:55,121 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:55,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:58:55,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53004 deadline: 1732539595130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:58:55,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:55,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:58:55,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:58:55,264 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-25T12:58:55,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:55,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. as already flushing 2024-11-25T12:58:55,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:55,265 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:55,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:55,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:58:55,369 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/9c90a2a1795a4b9e96f0d7051d82b3b4 2024-11-25T12:58:55,385 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/088697dd5b2f454cabff439e75bb2231 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/088697dd5b2f454cabff439e75bb2231 2024-11-25T12:58:55,389 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/088697dd5b2f454cabff439e75bb2231, entries=200, sequenceid=416, filesize=14.4 K 2024-11-25T12:58:55,390 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/e177698dad8049e0a2537e081f737519 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/e177698dad8049e0a2537e081f737519 2024-11-25T12:58:55,394 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/e177698dad8049e0a2537e081f737519, entries=150, sequenceid=416, filesize=12.0 K 2024-11-25T12:58:55,394 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/9c90a2a1795a4b9e96f0d7051d82b3b4 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/9c90a2a1795a4b9e96f0d7051d82b3b4 2024-11-25T12:58:55,398 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/9c90a2a1795a4b9e96f0d7051d82b3b4, entries=150, sequenceid=416, filesize=12.0 K 2024-11-25T12:58:55,399 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 21ea60aa3027cec780fb8ce3ec8872d3 in 927ms, sequenceid=416, compaction requested=true 2024-11-25T12:58:55,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 21ea60aa3027cec780fb8ce3ec8872d3: 2024-11-25T12:58:55,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21ea60aa3027cec780fb8ce3ec8872d3:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:58:55,399 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:58:55,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:58:55,399 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:58:55,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21ea60aa3027cec780fb8ce3ec8872d3:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:58:55,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:58:55,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21ea60aa3027cec780fb8ce3ec8872d3:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:58:55,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:58:55,400 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42669 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:58:55,400 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:58:55,400 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1541): 21ea60aa3027cec780fb8ce3ec8872d3/A is initiating minor compaction (all files) 2024-11-25T12:58:55,400 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): 21ea60aa3027cec780fb8ce3ec8872d3/B is initiating minor compaction (all files) 2024-11-25T12:58:55,400 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21ea60aa3027cec780fb8ce3ec8872d3/A in TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:55,400 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21ea60aa3027cec780fb8ce3ec8872d3/B in TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:55,400 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/f8576c377a5446b7933127ea9c6285fb, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/c7b57303fd954365b8e4cf9153d77bad, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/088697dd5b2f454cabff439e75bb2231] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp, totalSize=41.7 K 2024-11-25T12:58:55,400 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/7568500f045f4ab4af9ce2fa4aeac049, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/36dd56d3b12a4bf29c796aa0010e9719, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/e177698dad8049e0a2537e081f737519] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp, totalSize=36.9 K 2024-11-25T12:58:55,401 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8576c377a5446b7933127ea9c6285fb, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1732539531990 2024-11-25T12:58:55,401 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 7568500f045f4ab4af9ce2fa4aeac049, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1732539531990 2024-11-25T12:58:55,401 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7b57303fd954365b8e4cf9153d77bad, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1732539532640 2024-11-25T12:58:55,401 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 36dd56d3b12a4bf29c796aa0010e9719, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1732539532640 2024-11-25T12:58:55,401 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 088697dd5b2f454cabff439e75bb2231, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1732539533322 2024-11-25T12:58:55,401 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting e177698dad8049e0a2537e081f737519, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1732539533327 2024-11-25T12:58:55,407 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21ea60aa3027cec780fb8ce3ec8872d3#B#compaction#264 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:58:55,407 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/efd130c0852a49f7afb732b661e7899d is 50, key is test_row_0/B:col10/1732539534471/Put/seqid=0 2024-11-25T12:58:55,407 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21ea60aa3027cec780fb8ce3ec8872d3#A#compaction#265 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:58:55,408 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/f92af670cfb747699bf38b1bb85f7f05 is 50, key is test_row_0/A:col10/1732539534471/Put/seqid=0 2024-11-25T12:58:55,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742134_1310 (size=13289) 2024-11-25T12:58:55,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742135_1311 (size=13289) 2024-11-25T12:58:55,418 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-25T12:58:55,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:55,418 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 21ea60aa3027cec780fb8ce3ec8872d3 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-25T12:58:55,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/621ce9702dca4c50a80d4d024fd3095c is 50, key is test_row_0/A:col10/1732539534486/Put/seqid=0 2024-11-25T12:58:55,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742136_1312 (size=12301) 2024-11-25T12:58:55,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-25T12:58:55,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 21ea60aa3027cec780fb8ce3ec8872d3 2024-11-25T12:58:55,621 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. as already flushing 2024-11-25T12:58:55,621 INFO [Thread-1048 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1048. 2024-11-25T12:58:55,621 DEBUG [Thread-1048 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:58:55,621 DEBUG [Thread-1048 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:58:55,621 DEBUG [Thread-1048 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:58:55,621 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:58:55,631 INFO [Thread-1054 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1054. 2024-11-25T12:58:55,631 INFO [Thread-1050 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1050. 2024-11-25T12:58:55,631 DEBUG [Thread-1054 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:58:55,631 DEBUG [Thread-1050 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:58:55,631 DEBUG [Thread-1054 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:58:55,631 DEBUG [Thread-1050 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:58:55,631 DEBUG [Thread-1054 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:58:55,631 DEBUG [Thread-1050 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:58:55,631 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:58:55,631 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:58:55,640 INFO [Thread-1052 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1052. 2024-11-25T12:58:55,640 DEBUG [Thread-1052 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:58:55,640 DEBUG [Thread-1052 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:58:55,640 DEBUG [Thread-1052 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:58:55,640 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:58:55,814 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/efd130c0852a49f7afb732b661e7899d as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/efd130c0852a49f7afb732b661e7899d 2024-11-25T12:58:55,815 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/f92af670cfb747699bf38b1bb85f7f05 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/f92af670cfb747699bf38b1bb85f7f05 2024-11-25T12:58:55,818 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21ea60aa3027cec780fb8ce3ec8872d3/B of 21ea60aa3027cec780fb8ce3ec8872d3 into efd130c0852a49f7afb732b661e7899d(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:58:55,818 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21ea60aa3027cec780fb8ce3ec8872d3: 2024-11-25T12:58:55,818 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., storeName=21ea60aa3027cec780fb8ce3ec8872d3/B, priority=13, startTime=1732539535399; duration=0sec 2024-11-25T12:58:55,818 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:58:55,819 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21ea60aa3027cec780fb8ce3ec8872d3:B 2024-11-25T12:58:55,819 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21ea60aa3027cec780fb8ce3ec8872d3/A of 21ea60aa3027cec780fb8ce3ec8872d3 into f92af670cfb747699bf38b1bb85f7f05(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:58:55,819 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21ea60aa3027cec780fb8ce3ec8872d3: 2024-11-25T12:58:55,819 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., storeName=21ea60aa3027cec780fb8ce3ec8872d3/A, priority=13, startTime=1732539535399; duration=0sec 2024-11-25T12:58:55,819 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:58:55,819 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:58:55,819 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21ea60aa3027cec780fb8ce3ec8872d3:A 2024-11-25T12:58:55,819 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:58:55,819 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): 21ea60aa3027cec780fb8ce3ec8872d3/C is initiating minor compaction (all files) 2024-11-25T12:58:55,819 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21ea60aa3027cec780fb8ce3ec8872d3/C in TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:55,820 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/7e7278833b2b408591b586a25b68e1ef, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/d7fcc55c135448169c13d73e2ab21645, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/9c90a2a1795a4b9e96f0d7051d82b3b4] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp, totalSize=36.9 K 2024-11-25T12:58:55,820 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e7278833b2b408591b586a25b68e1ef, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1732539531990 2024-11-25T12:58:55,820 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting d7fcc55c135448169c13d73e2ab21645, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1732539532640 2024-11-25T12:58:55,820 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c90a2a1795a4b9e96f0d7051d82b3b4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1732539533327 2024-11-25T12:58:55,826 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/621ce9702dca4c50a80d4d024fd3095c 2024-11-25T12:58:55,826 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21ea60aa3027cec780fb8ce3ec8872d3#C#compaction#267 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:58:55,827 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/b5f121f0bc3c4d96a4525733de810bfe is 50, key is test_row_0/C:col10/1732539534471/Put/seqid=0 2024-11-25T12:58:55,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/fdead27102c947f387bce74c237d12f7 is 50, key is test_row_0/B:col10/1732539534486/Put/seqid=0 2024-11-25T12:58:55,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742137_1313 (size=13289) 2024-11-25T12:58:55,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742138_1314 (size=12301) 2024-11-25T12:58:55,844 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/fdead27102c947f387bce74c237d12f7 2024-11-25T12:58:55,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/efd7edec7ba64ab0a38c2d9a22d5661c is 50, key is test_row_0/C:col10/1732539534486/Put/seqid=0 2024-11-25T12:58:55,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742139_1315 (size=12301) 2024-11-25T12:58:56,253 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/b5f121f0bc3c4d96a4525733de810bfe as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/b5f121f0bc3c4d96a4525733de810bfe 2024-11-25T12:58:56,259 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/efd7edec7ba64ab0a38c2d9a22d5661c 2024-11-25T12:58:56,262 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21ea60aa3027cec780fb8ce3ec8872d3/C of 21ea60aa3027cec780fb8ce3ec8872d3 into b5f121f0bc3c4d96a4525733de810bfe(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:58:56,262 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21ea60aa3027cec780fb8ce3ec8872d3: 2024-11-25T12:58:56,262 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., storeName=21ea60aa3027cec780fb8ce3ec8872d3/C, priority=13, startTime=1732539535399; duration=0sec 2024-11-25T12:58:56,262 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:58:56,262 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21ea60aa3027cec780fb8ce3ec8872d3:C 2024-11-25T12:58:56,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/621ce9702dca4c50a80d4d024fd3095c as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/621ce9702dca4c50a80d4d024fd3095c 2024-11-25T12:58:56,271 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/621ce9702dca4c50a80d4d024fd3095c, entries=150, sequenceid=429, filesize=12.0 K 2024-11-25T12:58:56,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/fdead27102c947f387bce74c237d12f7 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/fdead27102c947f387bce74c237d12f7 2024-11-25T12:58:56,278 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/fdead27102c947f387bce74c237d12f7, entries=150, sequenceid=429, filesize=12.0 K 2024-11-25T12:58:56,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/efd7edec7ba64ab0a38c2d9a22d5661c as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/efd7edec7ba64ab0a38c2d9a22d5661c 2024-11-25T12:58:56,284 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/efd7edec7ba64ab0a38c2d9a22d5661c, entries=150, sequenceid=429, filesize=12.0 K 2024-11-25T12:58:56,285 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=26.84 KB/27480 for 21ea60aa3027cec780fb8ce3ec8872d3 in 867ms, sequenceid=429, compaction requested=false 2024-11-25T12:58:56,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 21ea60aa3027cec780fb8ce3ec8872d3: 2024-11-25T12:58:56,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:58:56,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-25T12:58:56,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-25T12:58:56,291 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-25T12:58:56,291 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9660 sec 2024-11-25T12:58:56,293 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.9730 sec 2024-11-25T12:58:56,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-25T12:58:56,470 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:58:57,355 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-25T12:59:04,742 INFO [Thread-1056 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1056. 2024-11-25T12:59:04,742 DEBUG [Thread-1056 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:59:04,742 DEBUG [Thread-1056 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:59:04,742 DEBUG [Thread-1056 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:59:04,742 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:59:04,743 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-25T12:59:04,743 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 84 2024-11-25T12:59:04,743 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 65 2024-11-25T12:59:04,743 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 74 2024-11-25T12:59:04,743 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 76 2024-11-25T12:59:04,743 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 44 2024-11-25T12:59:04,743 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-25T12:59:04,743 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2545 2024-11-25T12:59:04,743 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2536 2024-11-25T12:59:04,743 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2564 2024-11-25T12:59:04,743 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2566 2024-11-25T12:59:04,743 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2521 2024-11-25T12:59:04,743 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-25T12:59:04,743 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T12:59:04,743 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T12:59:04,743 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:59:04,743 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:59:04,743 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:59:04,744 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-25T12:59:04,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-25T12:59:04,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-25T12:59:04,747 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732539544747"}]},"ts":"1732539544747"} 2024-11-25T12:59:04,749 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-25T12:59:04,749 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-25T12:59:04,750 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-25T12:59:04,751 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=21ea60aa3027cec780fb8ce3ec8872d3, UNASSIGN}] 2024-11-25T12:59:04,752 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=21ea60aa3027cec780fb8ce3ec8872d3, UNASSIGN 2024-11-25T12:59:04,753 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=78 updating hbase:meta row=21ea60aa3027cec780fb8ce3ec8872d3, regionState=CLOSING, regionLocation=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:04,755 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=21ea60aa3027cec780fb8ce3ec8872d3, UNASSIGN because future has completed 2024-11-25T12:59:04,755 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-25T12:59:04,755 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; CloseRegionProcedure 21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246}] 2024-11-25T12:59:04,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-25T12:59:04,908 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] handler.UnassignRegionHandler(122): Close 21ea60aa3027cec780fb8ce3ec8872d3 2024-11-25T12:59:04,908 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-25T12:59:04,908 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1722): Closing 21ea60aa3027cec780fb8ce3ec8872d3, disabling compactions & flushes 2024-11-25T12:59:04,908 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:59:04,908 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:59:04,908 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. after waiting 0 ms 2024-11-25T12:59:04,908 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:59:04,908 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(2902): Flushing 21ea60aa3027cec780fb8ce3ec8872d3 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-25T12:59:04,912 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/9e20f1a800ce4c119139fa28a2c4f933 is 50, key is test_row_0/A:col10/1732539535639/Put/seqid=0 2024-11-25T12:59:04,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742140_1316 (size=12301) 2024-11-25T12:59:05,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-25T12:59:05,317 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/9e20f1a800ce4c119139fa28a2c4f933 2024-11-25T12:59:05,326 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/dfe3d0d65e4143e7b0385424d8cb8c4e is 50, key is test_row_0/B:col10/1732539535639/Put/seqid=0 2024-11-25T12:59:05,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742141_1317 (size=12301) 2024-11-25T12:59:05,352 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/dfe3d0d65e4143e7b0385424d8cb8c4e 2024-11-25T12:59:05,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-25T12:59:05,372 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/796926ef2e9c4f988187bec09eb81e21 is 50, key is test_row_0/C:col10/1732539535639/Put/seqid=0 2024-11-25T12:59:05,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742142_1318 (size=12301) 2024-11-25T12:59:05,382 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/796926ef2e9c4f988187bec09eb81e21 2024-11-25T12:59:05,390 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/A/9e20f1a800ce4c119139fa28a2c4f933 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/9e20f1a800ce4c119139fa28a2c4f933 2024-11-25T12:59:05,393 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/9e20f1a800ce4c119139fa28a2c4f933, entries=150, sequenceid=440, filesize=12.0 K 2024-11-25T12:59:05,396 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/B/dfe3d0d65e4143e7b0385424d8cb8c4e as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/dfe3d0d65e4143e7b0385424d8cb8c4e 2024-11-25T12:59:05,408 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/dfe3d0d65e4143e7b0385424d8cb8c4e, entries=150, sequenceid=440, filesize=12.0 K 2024-11-25T12:59:05,409 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/.tmp/C/796926ef2e9c4f988187bec09eb81e21 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/796926ef2e9c4f988187bec09eb81e21 2024-11-25T12:59:05,414 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/796926ef2e9c4f988187bec09eb81e21, entries=150, sequenceid=440, filesize=12.0 K 2024-11-25T12:59:05,415 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 21ea60aa3027cec780fb8ce3ec8872d3 in 507ms, sequenceid=440, compaction requested=true 2024-11-25T12:59:05,416 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/48b4926c83854b2690630b6c6b249609, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/64e408fb3daf47a89800635736b8f450, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/6e413a1d2b09485b897a50aa02c350ad, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/dabe892d4a434d8ea8a4eafe25b7097a, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/dcb716eb302d44afa8df29008645a28c, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/0b138a8aa27d4da6823bed5e6930c09b, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/34340069d0e043d4ac4ab64128a808f8, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/75b76a041c674850b02efce9543325ce, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/6aba27c71ffd4303a77806ca7a0b2dc1, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/2d800f37ccef494d805b39462dec1f19, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/2faf7ea9433641149a5cbe7122dfdae3, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/ae5fd395d52c49629cb33a5eddb474be, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/c4a9f0cc8814487a8bcf5e725f03ee01, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/4601ffae07694a84b04aa5ed1f453620, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/745f21900c6846b18f971a4449f2332c, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/f25ec1eae4064f318581c4559336d8ff, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/fd95edb09a3c4ed5b5c931d68023a341, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/1c2f52fa996e47d2b8db5cddfb8427d3, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/b27934e82ff5491e82f87e2d924b0245, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/a596aa19e80045c2b2a63c47c6652496, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/2760aadc3c6b45d4a231c9dd84b0bfcc, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/41f091d4fdb44dbea1b6f6a1b849c500, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/5bb8e5feb31d4fba867e8b3cd84662f1, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/aa44964cc6564e44bd6d04e082718c01, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/51773ac09fcb4bce8fb3eaf6426d75ca, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/f8576c377a5446b7933127ea9c6285fb, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/bb9f5828fb984363bb751a603a5c4bcb, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/c7b57303fd954365b8e4cf9153d77bad, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/088697dd5b2f454cabff439e75bb2231] to archive 2024-11-25T12:59:05,417 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T12:59:05,419 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/48b4926c83854b2690630b6c6b249609 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/48b4926c83854b2690630b6c6b249609 2024-11-25T12:59:05,420 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/64e408fb3daf47a89800635736b8f450 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/64e408fb3daf47a89800635736b8f450 2024-11-25T12:59:05,421 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/6e413a1d2b09485b897a50aa02c350ad to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/6e413a1d2b09485b897a50aa02c350ad 2024-11-25T12:59:05,422 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/dabe892d4a434d8ea8a4eafe25b7097a to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/dabe892d4a434d8ea8a4eafe25b7097a 2024-11-25T12:59:05,424 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/dcb716eb302d44afa8df29008645a28c to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/dcb716eb302d44afa8df29008645a28c 2024-11-25T12:59:05,425 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/0b138a8aa27d4da6823bed5e6930c09b to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/0b138a8aa27d4da6823bed5e6930c09b 2024-11-25T12:59:05,426 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/34340069d0e043d4ac4ab64128a808f8 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/34340069d0e043d4ac4ab64128a808f8 2024-11-25T12:59:05,427 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/75b76a041c674850b02efce9543325ce to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/75b76a041c674850b02efce9543325ce 2024-11-25T12:59:05,428 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/6aba27c71ffd4303a77806ca7a0b2dc1 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/6aba27c71ffd4303a77806ca7a0b2dc1 2024-11-25T12:59:05,430 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/2d800f37ccef494d805b39462dec1f19 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/2d800f37ccef494d805b39462dec1f19 2024-11-25T12:59:05,431 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/2faf7ea9433641149a5cbe7122dfdae3 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/2faf7ea9433641149a5cbe7122dfdae3 2024-11-25T12:59:05,433 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/ae5fd395d52c49629cb33a5eddb474be to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/ae5fd395d52c49629cb33a5eddb474be 2024-11-25T12:59:05,434 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/c4a9f0cc8814487a8bcf5e725f03ee01 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/c4a9f0cc8814487a8bcf5e725f03ee01 2024-11-25T12:59:05,435 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/4601ffae07694a84b04aa5ed1f453620 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/4601ffae07694a84b04aa5ed1f453620 2024-11-25T12:59:05,437 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/745f21900c6846b18f971a4449f2332c to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/745f21900c6846b18f971a4449f2332c 2024-11-25T12:59:05,438 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/f25ec1eae4064f318581c4559336d8ff to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/f25ec1eae4064f318581c4559336d8ff 2024-11-25T12:59:05,439 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/fd95edb09a3c4ed5b5c931d68023a341 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/fd95edb09a3c4ed5b5c931d68023a341 2024-11-25T12:59:05,453 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/1c2f52fa996e47d2b8db5cddfb8427d3 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/1c2f52fa996e47d2b8db5cddfb8427d3 2024-11-25T12:59:05,458 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/b27934e82ff5491e82f87e2d924b0245 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/b27934e82ff5491e82f87e2d924b0245 2024-11-25T12:59:05,459 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/a596aa19e80045c2b2a63c47c6652496 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/a596aa19e80045c2b2a63c47c6652496 2024-11-25T12:59:05,461 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/2760aadc3c6b45d4a231c9dd84b0bfcc to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/2760aadc3c6b45d4a231c9dd84b0bfcc 2024-11-25T12:59:05,462 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/41f091d4fdb44dbea1b6f6a1b849c500 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/41f091d4fdb44dbea1b6f6a1b849c500 2024-11-25T12:59:05,463 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/5bb8e5feb31d4fba867e8b3cd84662f1 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/5bb8e5feb31d4fba867e8b3cd84662f1 2024-11-25T12:59:05,465 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/aa44964cc6564e44bd6d04e082718c01 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/aa44964cc6564e44bd6d04e082718c01 2024-11-25T12:59:05,466 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/51773ac09fcb4bce8fb3eaf6426d75ca to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/51773ac09fcb4bce8fb3eaf6426d75ca 2024-11-25T12:59:05,467 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/f8576c377a5446b7933127ea9c6285fb to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/f8576c377a5446b7933127ea9c6285fb 2024-11-25T12:59:05,468 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/bb9f5828fb984363bb751a603a5c4bcb to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/bb9f5828fb984363bb751a603a5c4bcb 2024-11-25T12:59:05,469 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/c7b57303fd954365b8e4cf9153d77bad to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/c7b57303fd954365b8e4cf9153d77bad 2024-11-25T12:59:05,470 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/088697dd5b2f454cabff439e75bb2231 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/088697dd5b2f454cabff439e75bb2231 2024-11-25T12:59:05,485 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/eb655ad7125f412aae88a1f988b3b88c, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/70c0dc94245346a6976795c720879ce6, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/b452326c17ad45dc9b77bbdf2d1a6bbd, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/23581281edce4ea482676d8eca8ec78e, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/8fe50a49de2847608b4c4bc2cb0c34e3, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/44e1280f7b5c47988c86898d9492e6b1, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/7fab9161fe6443768b91bcafd461d957, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/4d6df410dbbf4b8da03e5ef6d6272fc6, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/d5e560ec7b2b4abd8c4df82cd0df8019, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/90725d57a4534cc18352a65d81f5b621, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/bb966ca48934440e9b0df545d41485d2, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/994b5e0d99ab449d98d613e1d2dffa89, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/9e21995419c34f8d9772ab9ae5311e65, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/cb857945b73044d1b14552c1b21c5479, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/b80b6c887965448d9ef0deedb3f2d717, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/1255251adaf94efc959aa997bbd12426, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/e40085c8dbec47f6912b3c4d47f97d2c, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/b562c6273f024a81ac4a40f95532dc28, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/f9b0fa0f43204160bc5c8ad1e4bb43f6, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/c4967385e672492c942ac047149c6e87, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/4eea3421ec53485aa60aed231cc906e8, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/17291e965c08450f977639620542f89c, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/34f0dee6793c4d74bf8248e0302cc04a, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/bea503c95cca49429b488345d735cea2, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/a95d5c54af604865891bd5a117b7e6e0, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/7568500f045f4ab4af9ce2fa4aeac049, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/26cc934c03204482b91e720bf60c5408, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/36dd56d3b12a4bf29c796aa0010e9719, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/e177698dad8049e0a2537e081f737519] to archive 2024-11-25T12:59:05,488 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T12:59:05,489 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/eb655ad7125f412aae88a1f988b3b88c to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/eb655ad7125f412aae88a1f988b3b88c 2024-11-25T12:59:05,493 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/70c0dc94245346a6976795c720879ce6 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/70c0dc94245346a6976795c720879ce6 2024-11-25T12:59:05,495 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/b452326c17ad45dc9b77bbdf2d1a6bbd to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/b452326c17ad45dc9b77bbdf2d1a6bbd 2024-11-25T12:59:05,496 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/23581281edce4ea482676d8eca8ec78e to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/23581281edce4ea482676d8eca8ec78e 2024-11-25T12:59:05,497 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/8fe50a49de2847608b4c4bc2cb0c34e3 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/8fe50a49de2847608b4c4bc2cb0c34e3 2024-11-25T12:59:05,498 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/44e1280f7b5c47988c86898d9492e6b1 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/44e1280f7b5c47988c86898d9492e6b1 2024-11-25T12:59:05,499 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/7fab9161fe6443768b91bcafd461d957 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/7fab9161fe6443768b91bcafd461d957 2024-11-25T12:59:05,500 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/4d6df410dbbf4b8da03e5ef6d6272fc6 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/4d6df410dbbf4b8da03e5ef6d6272fc6 2024-11-25T12:59:05,501 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/d5e560ec7b2b4abd8c4df82cd0df8019 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/d5e560ec7b2b4abd8c4df82cd0df8019 2024-11-25T12:59:05,502 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/90725d57a4534cc18352a65d81f5b621 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/90725d57a4534cc18352a65d81f5b621 2024-11-25T12:59:05,503 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/bb966ca48934440e9b0df545d41485d2 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/bb966ca48934440e9b0df545d41485d2 2024-11-25T12:59:05,504 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/994b5e0d99ab449d98d613e1d2dffa89 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/994b5e0d99ab449d98d613e1d2dffa89 2024-11-25T12:59:05,505 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/9e21995419c34f8d9772ab9ae5311e65 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/9e21995419c34f8d9772ab9ae5311e65 2024-11-25T12:59:05,506 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/cb857945b73044d1b14552c1b21c5479 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/cb857945b73044d1b14552c1b21c5479 2024-11-25T12:59:05,507 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/b80b6c887965448d9ef0deedb3f2d717 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/b80b6c887965448d9ef0deedb3f2d717 2024-11-25T12:59:05,508 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/1255251adaf94efc959aa997bbd12426 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/1255251adaf94efc959aa997bbd12426 2024-11-25T12:59:05,509 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/e40085c8dbec47f6912b3c4d47f97d2c to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/e40085c8dbec47f6912b3c4d47f97d2c 2024-11-25T12:59:05,510 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/b562c6273f024a81ac4a40f95532dc28 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/b562c6273f024a81ac4a40f95532dc28 2024-11-25T12:59:05,512 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/f9b0fa0f43204160bc5c8ad1e4bb43f6 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/f9b0fa0f43204160bc5c8ad1e4bb43f6 2024-11-25T12:59:05,513 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/c4967385e672492c942ac047149c6e87 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/c4967385e672492c942ac047149c6e87 2024-11-25T12:59:05,514 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/4eea3421ec53485aa60aed231cc906e8 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/4eea3421ec53485aa60aed231cc906e8 2024-11-25T12:59:05,515 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/17291e965c08450f977639620542f89c to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/17291e965c08450f977639620542f89c 2024-11-25T12:59:05,516 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/34f0dee6793c4d74bf8248e0302cc04a to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/34f0dee6793c4d74bf8248e0302cc04a 2024-11-25T12:59:05,517 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/bea503c95cca49429b488345d735cea2 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/bea503c95cca49429b488345d735cea2 2024-11-25T12:59:05,518 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/a95d5c54af604865891bd5a117b7e6e0 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/a95d5c54af604865891bd5a117b7e6e0 2024-11-25T12:59:05,519 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/7568500f045f4ab4af9ce2fa4aeac049 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/7568500f045f4ab4af9ce2fa4aeac049 2024-11-25T12:59:05,522 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/26cc934c03204482b91e720bf60c5408 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/26cc934c03204482b91e720bf60c5408 2024-11-25T12:59:05,527 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/36dd56d3b12a4bf29c796aa0010e9719 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/36dd56d3b12a4bf29c796aa0010e9719 2024-11-25T12:59:05,528 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/e177698dad8049e0a2537e081f737519 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/e177698dad8049e0a2537e081f737519 2024-11-25T12:59:05,535 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/7275d66c4c0b4ba397a86c10ddbb6a0b, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/9d2b1a3435404800ad54f6bbe784b524, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/f020af86f275434a845156e51ca1064b, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/6b9119d676114d96a43ae469a51a6c08, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/7c2c487f385b473ea216253f051ab9f8, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/2ffb2f8c2f044c2581207e18c41902d5, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/3c5035de040d42dba55c5d00667bc2ec, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/bf065fa4665d405090357f1f36923745, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/819802ee4c744f3492f71ead70041616, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/686367e550324a5c95b396a31f59078a, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/006162a661c1447c94528798ca6c200b, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/8f002ff1b2174d8882df3e86be63285f, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/651da5d107c848599d6d44553bb82389, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/126258b976a147e8b9c8fdcf5281bab5, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/6442bf90c5364fd2ae6185b72d8debd8, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/b5477000f70c477fbb4283c04ac1e3e5, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/0bc07e18a67644bf83cd6eeb3674063c, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/022755c065b943ef91f833a5c86fd5a9, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/357988c2337e450da1d1a763bd4fad0a, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/d11ea32e39024acc8365a535d08a0f6e, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/3d677843a89a4673a9754e6d112e795a, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/994102e79f8f4f97a2807a588bd65a56, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/5912802b9067463da6d2fe35d7451136, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/f1ec1940cbfa43dbba0cf36e5c381950, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/af49e2d083bf4a3c96bd2425db26a8a6, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/7e7278833b2b408591b586a25b68e1ef, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/36ce8dde402c4d24a0dc0ce8249cbdb6, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/d7fcc55c135448169c13d73e2ab21645, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/9c90a2a1795a4b9e96f0d7051d82b3b4] to archive 2024-11-25T12:59:05,537 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T12:59:05,539 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/7275d66c4c0b4ba397a86c10ddbb6a0b to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/7275d66c4c0b4ba397a86c10ddbb6a0b 2024-11-25T12:59:05,540 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/9d2b1a3435404800ad54f6bbe784b524 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/9d2b1a3435404800ad54f6bbe784b524 2024-11-25T12:59:05,541 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/f020af86f275434a845156e51ca1064b to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/f020af86f275434a845156e51ca1064b 2024-11-25T12:59:05,542 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/6b9119d676114d96a43ae469a51a6c08 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/6b9119d676114d96a43ae469a51a6c08 2024-11-25T12:59:05,544 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/7c2c487f385b473ea216253f051ab9f8 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/7c2c487f385b473ea216253f051ab9f8 2024-11-25T12:59:05,545 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/2ffb2f8c2f044c2581207e18c41902d5 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/2ffb2f8c2f044c2581207e18c41902d5 2024-11-25T12:59:05,552 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/3c5035de040d42dba55c5d00667bc2ec to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/3c5035de040d42dba55c5d00667bc2ec 2024-11-25T12:59:05,554 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/bf065fa4665d405090357f1f36923745 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/bf065fa4665d405090357f1f36923745 2024-11-25T12:59:05,555 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/819802ee4c744f3492f71ead70041616 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/819802ee4c744f3492f71ead70041616 2024-11-25T12:59:05,556 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/686367e550324a5c95b396a31f59078a to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/686367e550324a5c95b396a31f59078a 2024-11-25T12:59:05,559 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/006162a661c1447c94528798ca6c200b to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/006162a661c1447c94528798ca6c200b 2024-11-25T12:59:05,560 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/8f002ff1b2174d8882df3e86be63285f to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/8f002ff1b2174d8882df3e86be63285f 2024-11-25T12:59:05,561 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/651da5d107c848599d6d44553bb82389 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/651da5d107c848599d6d44553bb82389 2024-11-25T12:59:05,562 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/126258b976a147e8b9c8fdcf5281bab5 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/126258b976a147e8b9c8fdcf5281bab5 2024-11-25T12:59:05,564 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/6442bf90c5364fd2ae6185b72d8debd8 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/6442bf90c5364fd2ae6185b72d8debd8 2024-11-25T12:59:05,565 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/b5477000f70c477fbb4283c04ac1e3e5 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/b5477000f70c477fbb4283c04ac1e3e5 2024-11-25T12:59:05,566 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/0bc07e18a67644bf83cd6eeb3674063c to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/0bc07e18a67644bf83cd6eeb3674063c 2024-11-25T12:59:05,567 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/022755c065b943ef91f833a5c86fd5a9 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/022755c065b943ef91f833a5c86fd5a9 2024-11-25T12:59:05,578 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/357988c2337e450da1d1a763bd4fad0a to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/357988c2337e450da1d1a763bd4fad0a 2024-11-25T12:59:05,588 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/d11ea32e39024acc8365a535d08a0f6e to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/d11ea32e39024acc8365a535d08a0f6e 2024-11-25T12:59:05,590 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/3d677843a89a4673a9754e6d112e795a to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/3d677843a89a4673a9754e6d112e795a 2024-11-25T12:59:05,592 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/994102e79f8f4f97a2807a588bd65a56 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/994102e79f8f4f97a2807a588bd65a56 2024-11-25T12:59:05,597 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/5912802b9067463da6d2fe35d7451136 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/5912802b9067463da6d2fe35d7451136 2024-11-25T12:59:05,601 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/f1ec1940cbfa43dbba0cf36e5c381950 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/f1ec1940cbfa43dbba0cf36e5c381950 2024-11-25T12:59:05,611 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/af49e2d083bf4a3c96bd2425db26a8a6 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/af49e2d083bf4a3c96bd2425db26a8a6 2024-11-25T12:59:05,624 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/7e7278833b2b408591b586a25b68e1ef to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/7e7278833b2b408591b586a25b68e1ef 2024-11-25T12:59:05,630 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/36ce8dde402c4d24a0dc0ce8249cbdb6 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/36ce8dde402c4d24a0dc0ce8249cbdb6 2024-11-25T12:59:05,643 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/d7fcc55c135448169c13d73e2ab21645 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/d7fcc55c135448169c13d73e2ab21645 2024-11-25T12:59:05,652 DEBUG [StoreCloser-TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/9c90a2a1795a4b9e96f0d7051d82b3b4 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/9c90a2a1795a4b9e96f0d7051d82b3b4 2024-11-25T12:59:05,657 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/recovered.edits/443.seqid, newMaxSeqId=443, maxSeqId=1 2024-11-25T12:59:05,657 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. 2024-11-25T12:59:05,658 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1676): Region close journal for 21ea60aa3027cec780fb8ce3ec8872d3: Waiting for close lock at 1732539544908Running coprocessor pre-close hooks at 1732539544908Disabling compacts and flushes for region at 1732539544908Disabling writes for close at 1732539544908Obtaining lock to block concurrent updates at 1732539544908Preparing flush snapshotting stores in 21ea60aa3027cec780fb8ce3ec8872d3 at 1732539544908Finished memstore snapshotting TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1732539544908Flushing stores of TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3. at 1732539544909 (+1 ms)Flushing 21ea60aa3027cec780fb8ce3ec8872d3/A: creating writer at 1732539544909Flushing 21ea60aa3027cec780fb8ce3ec8872d3/A: appending metadata at 1732539544912 (+3 ms)Flushing 21ea60aa3027cec780fb8ce3ec8872d3/A: closing flushed file at 1732539544912Flushing 21ea60aa3027cec780fb8ce3ec8872d3/B: creating writer at 1732539545323 (+411 ms)Flushing 21ea60aa3027cec780fb8ce3ec8872d3/B: appending metadata at 1732539545326 (+3 ms)Flushing 21ea60aa3027cec780fb8ce3ec8872d3/B: closing flushed file at 1732539545326Flushing 21ea60aa3027cec780fb8ce3ec8872d3/C: creating writer at 1732539545360 (+34 ms)Flushing 21ea60aa3027cec780fb8ce3ec8872d3/C: appending metadata at 1732539545372 (+12 ms)Flushing 21ea60aa3027cec780fb8ce3ec8872d3/C: closing flushed file at 1732539545372Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7594279a: reopening flushed file at 1732539545389 (+17 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5595586d: reopening flushed file at 1732539545394 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5fbc0fd4: reopening flushed file at 1732539545408 (+14 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 21ea60aa3027cec780fb8ce3ec8872d3 in 507ms, sequenceid=440, compaction requested=true at 1732539545415 (+7 ms)Writing region close event to WAL at 1732539545654 (+239 ms)Running coprocessor post-close hooks at 1732539545657 (+3 ms)Closed at 1732539545657 2024-11-25T12:59:05,661 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=78 updating hbase:meta row=21ea60aa3027cec780fb8ce3ec8872d3, regionState=CLOSED 2024-11-25T12:59:05,664 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] handler.UnassignRegionHandler(157): Closed 21ea60aa3027cec780fb8ce3ec8872d3 2024-11-25T12:59:05,664 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=79, ppid=78, state=RUNNABLE, hasLock=false; CloseRegionProcedure 21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 because future has completed 2024-11-25T12:59:05,667 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-25T12:59:05,667 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; CloseRegionProcedure 21ea60aa3027cec780fb8ce3ec8872d3, server=1ff8009e463e,35905,1732539449246 in 910 msec 2024-11-25T12:59:05,669 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=78, resume processing ppid=77 2024-11-25T12:59:05,669 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, ppid=77, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=21ea60aa3027cec780fb8ce3ec8872d3, UNASSIGN in 916 msec 2024-11-25T12:59:05,672 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-25T12:59:05,672 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 920 msec 2024-11-25T12:59:05,677 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732539545676"}]},"ts":"1732539545676"} 2024-11-25T12:59:05,678 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-25T12:59:05,679 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-25T12:59:05,681 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 935 msec 2024-11-25T12:59:05,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-25T12:59:05,880 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:59:05,880 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-25T12:59:05,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T12:59:05,881 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=80, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T12:59:05,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-25T12:59:05,882 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=80, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T12:59:05,883 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3 2024-11-25T12:59:05,885 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A, FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B, FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C, FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/recovered.edits] 2024-11-25T12:59:05,887 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/621ce9702dca4c50a80d4d024fd3095c to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/621ce9702dca4c50a80d4d024fd3095c 2024-11-25T12:59:05,888 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/9e20f1a800ce4c119139fa28a2c4f933 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/9e20f1a800ce4c119139fa28a2c4f933 2024-11-25T12:59:05,889 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/f92af670cfb747699bf38b1bb85f7f05 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/A/f92af670cfb747699bf38b1bb85f7f05 2024-11-25T12:59:05,892 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/dfe3d0d65e4143e7b0385424d8cb8c4e to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/dfe3d0d65e4143e7b0385424d8cb8c4e 2024-11-25T12:59:05,893 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/efd130c0852a49f7afb732b661e7899d to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/efd130c0852a49f7afb732b661e7899d 2024-11-25T12:59:05,894 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/fdead27102c947f387bce74c237d12f7 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/B/fdead27102c947f387bce74c237d12f7 2024-11-25T12:59:05,899 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/796926ef2e9c4f988187bec09eb81e21 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/796926ef2e9c4f988187bec09eb81e21 2024-11-25T12:59:05,918 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/b5f121f0bc3c4d96a4525733de810bfe to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/b5f121f0bc3c4d96a4525733de810bfe 2024-11-25T12:59:05,920 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/efd7edec7ba64ab0a38c2d9a22d5661c to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/C/efd7edec7ba64ab0a38c2d9a22d5661c 2024-11-25T12:59:05,924 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/recovered.edits/443.seqid to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3/recovered.edits/443.seqid 2024-11-25T12:59:05,925 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/21ea60aa3027cec780fb8ce3ec8872d3 2024-11-25T12:59:05,925 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-25T12:59:05,929 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=80, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T12:59:05,941 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-25T12:59:05,943 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-25T12:59:05,945 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=80, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T12:59:05,945 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-25T12:59:05,945 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732539545945"}]},"ts":"9223372036854775807"} 2024-11-25T12:59:05,947 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-25T12:59:05,947 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 21ea60aa3027cec780fb8ce3ec8872d3, NAME => 'TestAcidGuarantees,,1732539513872.21ea60aa3027cec780fb8ce3ec8872d3.', STARTKEY => '', ENDKEY => ''}] 2024-11-25T12:59:05,947 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-25T12:59:05,947 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732539545947"}]},"ts":"9223372036854775807"} 2024-11-25T12:59:05,949 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-25T12:59:05,949 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=80, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T12:59:05,951 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 69 msec 2024-11-25T12:59:05,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-25T12:59:05,990 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-25T12:59:05,993 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:59:05,999 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=236 (was 238), OpenFileDescriptor=447 (was 455), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=944 (was 880) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1010 (was 1252) 2024-11-25T12:59:06,006 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=236, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=944, ProcessCount=11, AvailableMemoryMB=1010 2024-11-25T12:59:06,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-25T12:59:06,008 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T12:59:06,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] procedure2.ProcedureExecutor(1139): Stored pid=81, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-25T12:59:06,010 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T12:59:06,010 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 81 2024-11-25T12:59:06,010 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:06,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-11-25T12:59:06,011 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T12:59:06,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742143_1319 (size=959) 2024-11-25T12:59:06,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-11-25T12:59:06,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-11-25T12:59:06,417 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a 2024-11-25T12:59:06,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742144_1320 (size=53) 2024-11-25T12:59:06,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-11-25T12:59:06,822 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:59:06,822 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing de45cbd8ecc09a85a2c9aae45053021b, disabling compactions & flushes 2024-11-25T12:59:06,822 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:06,822 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:06,822 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. after waiting 0 ms 2024-11-25T12:59:06,822 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:06,822 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:06,822 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for de45cbd8ecc09a85a2c9aae45053021b: Waiting for close lock at 1732539546822Disabling compacts and flushes for region at 1732539546822Disabling writes for close at 1732539546822Writing region close event to WAL at 1732539546822Closed at 1732539546822 2024-11-25T12:59:06,823 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T12:59:06,823 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732539546823"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732539546823"}]},"ts":"1732539546823"} 2024-11-25T12:59:06,825 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T12:59:06,825 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T12:59:06,826 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732539546826"}]},"ts":"1732539546826"} 2024-11-25T12:59:06,827 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-25T12:59:06,827 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=de45cbd8ecc09a85a2c9aae45053021b, ASSIGN}] 2024-11-25T12:59:06,828 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=de45cbd8ecc09a85a2c9aae45053021b, ASSIGN 2024-11-25T12:59:06,828 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=de45cbd8ecc09a85a2c9aae45053021b, ASSIGN; state=OFFLINE, location=1ff8009e463e,35905,1732539449246; forceNewPlan=false, retain=false 2024-11-25T12:59:06,979 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=de45cbd8ecc09a85a2c9aae45053021b, regionState=OPENING, regionLocation=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:06,981 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=de45cbd8ecc09a85a2c9aae45053021b, ASSIGN because future has completed 2024-11-25T12:59:06,981 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; OpenRegionProcedure de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246}] 2024-11-25T12:59:07,135 INFO [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:07,136 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(7752): Opening region: {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} 2024-11-25T12:59:07,136 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:07,136 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:59:07,136 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(7794): checking encryption for de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:07,136 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(7797): checking classloading for de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:07,137 INFO [StoreOpener-de45cbd8ecc09a85a2c9aae45053021b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:07,139 INFO [StoreOpener-de45cbd8ecc09a85a2c9aae45053021b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region de45cbd8ecc09a85a2c9aae45053021b columnFamilyName A 2024-11-25T12:59:07,139 DEBUG [StoreOpener-de45cbd8ecc09a85a2c9aae45053021b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:07,139 INFO [StoreOpener-de45cbd8ecc09a85a2c9aae45053021b-1 {}] regionserver.HStore(327): Store=de45cbd8ecc09a85a2c9aae45053021b/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:59:07,139 INFO [StoreOpener-de45cbd8ecc09a85a2c9aae45053021b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:07,140 INFO [StoreOpener-de45cbd8ecc09a85a2c9aae45053021b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region de45cbd8ecc09a85a2c9aae45053021b columnFamilyName B 2024-11-25T12:59:07,140 DEBUG [StoreOpener-de45cbd8ecc09a85a2c9aae45053021b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:07,140 INFO [StoreOpener-de45cbd8ecc09a85a2c9aae45053021b-1 {}] regionserver.HStore(327): Store=de45cbd8ecc09a85a2c9aae45053021b/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:59:07,140 INFO [StoreOpener-de45cbd8ecc09a85a2c9aae45053021b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:07,141 INFO [StoreOpener-de45cbd8ecc09a85a2c9aae45053021b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region de45cbd8ecc09a85a2c9aae45053021b columnFamilyName C 2024-11-25T12:59:07,141 DEBUG [StoreOpener-de45cbd8ecc09a85a2c9aae45053021b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:07,142 INFO [StoreOpener-de45cbd8ecc09a85a2c9aae45053021b-1 {}] regionserver.HStore(327): Store=de45cbd8ecc09a85a2c9aae45053021b/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:59:07,142 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1038): replaying wal for de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:07,142 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:07,142 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:07,143 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1048): stopping wal replay for de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:07,143 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1060): Cleaning up temporary data for de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:07,143 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-25T12:59:07,144 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1093): writing seq id for de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:07,145 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T12:59:07,146 INFO [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1114): Opened de45cbd8ecc09a85a2c9aae45053021b; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69297641, jitterRate=0.03261531889438629}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-25T12:59:07,146 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1122): Running coprocessor post-open hooks for de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:07,146 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1006): Region open journal for de45cbd8ecc09a85a2c9aae45053021b: Running coprocessor pre-open hook at 1732539547136Writing region info on filesystem at 1732539547136Initializing all the Stores at 1732539547137 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732539547137Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732539547137Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732539547137Cleaning up temporary data from old regions at 1732539547143 (+6 ms)Running coprocessor post-open hooks at 1732539547146 (+3 ms)Region opened successfully at 1732539547146 2024-11-25T12:59:07,147 INFO [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., pid=83, masterSystemTime=1732539547133 2024-11-25T12:59:07,148 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:07,148 INFO [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:07,149 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=de45cbd8ecc09a85a2c9aae45053021b, regionState=OPEN, openSeqNum=2, regionLocation=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:07,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-11-25T12:59:07,150 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=83, ppid=82, state=RUNNABLE, hasLock=false; OpenRegionProcedure de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 because future has completed 2024-11-25T12:59:07,151 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-25T12:59:07,151 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; OpenRegionProcedure de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 in 169 msec 2024-11-25T12:59:07,153 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=82, resume processing ppid=81 2024-11-25T12:59:07,153 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, ppid=81, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=de45cbd8ecc09a85a2c9aae45053021b, ASSIGN in 324 msec 2024-11-25T12:59:07,153 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T12:59:07,154 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732539547153"}]},"ts":"1732539547153"} 2024-11-25T12:59:07,155 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-25T12:59:07,155 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T12:59:07,156 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1470 sec 2024-11-25T12:59:08,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-11-25T12:59:08,161 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:59:08,162 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15f86bdc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:59:08,162 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ff8009e463e,46289,-1 for getting cluster id 2024-11-25T12:59:08,163 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:59:08,164 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb972fd5-cdcb-422b-acfe-41fc80d301d9' 2024-11-25T12:59:08,164 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:59:08,164 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb972fd5-cdcb-422b-acfe-41fc80d301d9" 2024-11-25T12:59:08,164 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6902ec95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:59:08,164 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ff8009e463e,46289,-1] 2024-11-25T12:59:08,165 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:59:08,165 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:59:08,166 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59170, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:59:08,166 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f5d9d04, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:59:08,167 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:59:08,167 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,35905,1732539449246, seqNum=-1] 2024-11-25T12:59:08,168 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:59:08,168 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46460, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:59:08,169 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-25T12:59:08,170 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 1ff8009e463e,46289,1732539448369 2024-11-25T12:59:08,170 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@701a0230 2024-11-25T12:59:08,170 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-25T12:59:08,171 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59174, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-25T12:59:08,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-25T12:59:08,172 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T12:59:08,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-25T12:59:08,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742145_1321 (size=995) 2024-11-25T12:59:08,191 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-25T12:59:08,191 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-25T12:59:08,193 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-25T12:59:08,200 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=de45cbd8ecc09a85a2c9aae45053021b, REOPEN/MOVE}] 2024-11-25T12:59:08,201 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=de45cbd8ecc09a85a2c9aae45053021b, REOPEN/MOVE 2024-11-25T12:59:08,205 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=de45cbd8ecc09a85a2c9aae45053021b, regionState=CLOSING, regionLocation=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:08,207 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=de45cbd8ecc09a85a2c9aae45053021b, REOPEN/MOVE because future has completed 2024-11-25T12:59:08,207 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-25T12:59:08,208 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246}] 2024-11-25T12:59:08,361 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(122): Close de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:08,361 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-25T12:59:08,361 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1722): Closing de45cbd8ecc09a85a2c9aae45053021b, disabling compactions & flushes 2024-11-25T12:59:08,361 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:08,361 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:08,361 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. after waiting 0 ms 2024-11-25T12:59:08,361 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:08,365 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-25T12:59:08,366 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:08,366 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1676): Region close journal for de45cbd8ecc09a85a2c9aae45053021b: Waiting for close lock at 1732539548361Running coprocessor pre-close hooks at 1732539548361Disabling compacts and flushes for region at 1732539548361Disabling writes for close at 1732539548361Writing region close event to WAL at 1732539548361Running coprocessor post-close hooks at 1732539548366 (+5 ms)Closed at 1732539548366 2024-11-25T12:59:08,366 WARN [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionServer(3299): Not adding moved region record: de45cbd8ecc09a85a2c9aae45053021b to self. 2024-11-25T12:59:08,369 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(157): Closed de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:08,369 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=de45cbd8ecc09a85a2c9aae45053021b, regionState=CLOSED 2024-11-25T12:59:08,371 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 because future has completed 2024-11-25T12:59:08,374 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-25T12:59:08,374 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; CloseRegionProcedure de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 in 165 msec 2024-11-25T12:59:08,375 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=de45cbd8ecc09a85a2c9aae45053021b, REOPEN/MOVE; state=CLOSED, location=1ff8009e463e,35905,1732539449246; forceNewPlan=false, retain=true 2024-11-25T12:59:08,525 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=de45cbd8ecc09a85a2c9aae45053021b, regionState=OPENING, regionLocation=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:08,527 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=de45cbd8ecc09a85a2c9aae45053021b, REOPEN/MOVE because future has completed 2024-11-25T12:59:08,527 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246}] 2024-11-25T12:59:08,681 INFO [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:08,682 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(7752): Opening region: {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} 2024-11-25T12:59:08,682 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:08,682 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:59:08,682 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(7794): checking encryption for de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:08,682 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(7797): checking classloading for de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:08,686 INFO [StoreOpener-de45cbd8ecc09a85a2c9aae45053021b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:08,687 INFO [StoreOpener-de45cbd8ecc09a85a2c9aae45053021b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region de45cbd8ecc09a85a2c9aae45053021b columnFamilyName A 2024-11-25T12:59:08,688 DEBUG [StoreOpener-de45cbd8ecc09a85a2c9aae45053021b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:08,688 INFO [StoreOpener-de45cbd8ecc09a85a2c9aae45053021b-1 {}] regionserver.HStore(327): Store=de45cbd8ecc09a85a2c9aae45053021b/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:59:08,689 INFO [StoreOpener-de45cbd8ecc09a85a2c9aae45053021b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:08,689 INFO [StoreOpener-de45cbd8ecc09a85a2c9aae45053021b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region de45cbd8ecc09a85a2c9aae45053021b columnFamilyName B 2024-11-25T12:59:08,689 DEBUG [StoreOpener-de45cbd8ecc09a85a2c9aae45053021b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:08,690 INFO [StoreOpener-de45cbd8ecc09a85a2c9aae45053021b-1 {}] regionserver.HStore(327): Store=de45cbd8ecc09a85a2c9aae45053021b/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:59:08,690 INFO [StoreOpener-de45cbd8ecc09a85a2c9aae45053021b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:08,690 INFO [StoreOpener-de45cbd8ecc09a85a2c9aae45053021b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region de45cbd8ecc09a85a2c9aae45053021b columnFamilyName C 2024-11-25T12:59:08,690 DEBUG [StoreOpener-de45cbd8ecc09a85a2c9aae45053021b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:08,691 INFO [StoreOpener-de45cbd8ecc09a85a2c9aae45053021b-1 {}] regionserver.HStore(327): Store=de45cbd8ecc09a85a2c9aae45053021b/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:59:08,691 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1038): replaying wal for de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:08,692 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:08,693 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:08,693 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1048): stopping wal replay for de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:08,693 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1060): Cleaning up temporary data for de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:08,694 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-25T12:59:08,694 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1093): writing seq id for de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:08,695 INFO [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1114): Opened de45cbd8ecc09a85a2c9aae45053021b; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66106031, jitterRate=-0.014943376183509827}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-25T12:59:08,695 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1122): Running coprocessor post-open hooks for de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:08,696 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1006): Region open journal for de45cbd8ecc09a85a2c9aae45053021b: Running coprocessor pre-open hook at 1732539548682Writing region info on filesystem at 1732539548682Initializing all the Stores at 1732539548683 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732539548683Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732539548686 (+3 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732539548686Cleaning up temporary data from old regions at 1732539548693 (+7 ms)Running coprocessor post-open hooks at 1732539548695 (+2 ms)Region opened successfully at 1732539548696 (+1 ms) 2024-11-25T12:59:08,696 INFO [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., pid=88, masterSystemTime=1732539548679 2024-11-25T12:59:08,698 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:08,698 INFO [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:08,698 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=de45cbd8ecc09a85a2c9aae45053021b, regionState=OPEN, openSeqNum=5, regionLocation=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:08,699 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 because future has completed 2024-11-25T12:59:08,701 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=86 2024-11-25T12:59:08,701 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=86, state=SUCCESS, hasLock=false; OpenRegionProcedure de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 in 173 msec 2024-11-25T12:59:08,702 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-11-25T12:59:08,703 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=de45cbd8ecc09a85a2c9aae45053021b, REOPEN/MOVE in 501 msec 2024-11-25T12:59:08,705 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-25T12:59:08,705 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 510 msec 2024-11-25T12:59:08,706 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 532 msec 2024-11-25T12:59:08,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-25T12:59:08,707 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:59:08,708 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a739f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:59:08,708 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ff8009e463e,46289,-1 for getting cluster id 2024-11-25T12:59:08,708 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:59:08,709 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb972fd5-cdcb-422b-acfe-41fc80d301d9' 2024-11-25T12:59:08,709 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:59:08,709 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb972fd5-cdcb-422b-acfe-41fc80d301d9" 2024-11-25T12:59:08,709 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@126fc226, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:59:08,709 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ff8009e463e,46289,-1] 2024-11-25T12:59:08,709 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:59:08,709 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:59:08,710 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59194, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:59:08,710 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@120ea112, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:59:08,711 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@87fd48f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:59:08,711 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ff8009e463e,46289,-1 for getting cluster id 2024-11-25T12:59:08,711 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:59:08,712 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb972fd5-cdcb-422b-acfe-41fc80d301d9' 2024-11-25T12:59:08,712 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:59:08,712 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb972fd5-cdcb-422b-acfe-41fc80d301d9" 2024-11-25T12:59:08,712 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@988caa8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:59:08,712 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ff8009e463e,46289,-1] 2024-11-25T12:59:08,712 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:59:08,712 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:59:08,713 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59216, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:59:08,713 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c79704a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:59:08,714 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f0bd6a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:59:08,714 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ff8009e463e,46289,-1 for getting cluster id 2024-11-25T12:59:08,714 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:59:08,715 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb972fd5-cdcb-422b-acfe-41fc80d301d9' 2024-11-25T12:59:08,715 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:59:08,715 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb972fd5-cdcb-422b-acfe-41fc80d301d9" 2024-11-25T12:59:08,715 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@757c5bcc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:59:08,715 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ff8009e463e,46289,-1] 2024-11-25T12:59:08,715 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:59:08,715 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:59:08,716 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59232, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:59:08,716 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d4b0fb6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:59:08,717 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72c86b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:59:08,717 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ff8009e463e,46289,-1 for getting cluster id 2024-11-25T12:59:08,717 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:59:08,717 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb972fd5-cdcb-422b-acfe-41fc80d301d9' 2024-11-25T12:59:08,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:59:08,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb972fd5-cdcb-422b-acfe-41fc80d301d9" 2024-11-25T12:59:08,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7518936b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:59:08,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ff8009e463e,46289,-1] 2024-11-25T12:59:08,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:59:08,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:59:08,719 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59264, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:59:08,719 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17cd2c48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:59:08,721 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35aa2c8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:59:08,721 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ff8009e463e,46289,-1 for getting cluster id 2024-11-25T12:59:08,721 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:59:08,722 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb972fd5-cdcb-422b-acfe-41fc80d301d9' 2024-11-25T12:59:08,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:59:08,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb972fd5-cdcb-422b-acfe-41fc80d301d9" 2024-11-25T12:59:08,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@194b7e12, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:59:08,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ff8009e463e,46289,-1] 2024-11-25T12:59:08,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:59:08,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:59:08,723 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59286, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:59:08,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4922790, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:59:08,726 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c9deaeb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:59:08,726 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ff8009e463e,46289,-1 for getting cluster id 2024-11-25T12:59:08,726 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:59:08,726 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb972fd5-cdcb-422b-acfe-41fc80d301d9' 2024-11-25T12:59:08,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:59:08,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb972fd5-cdcb-422b-acfe-41fc80d301d9" 2024-11-25T12:59:08,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d6d675b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:59:08,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ff8009e463e,46289,-1] 2024-11-25T12:59:08,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:59:08,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:59:08,728 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59290, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:59:08,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f5fb3aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:59:08,728 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33f8ca2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:59:08,728 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ff8009e463e,46289,-1 for getting cluster id 2024-11-25T12:59:08,729 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:59:08,729 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb972fd5-cdcb-422b-acfe-41fc80d301d9' 2024-11-25T12:59:08,729 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:59:08,729 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb972fd5-cdcb-422b-acfe-41fc80d301d9" 2024-11-25T12:59:08,729 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b4b3759, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:59:08,729 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ff8009e463e,46289,-1] 2024-11-25T12:59:08,730 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:59:08,730 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:59:08,730 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59312, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:59:08,731 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69b1ad6c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:59:08,735 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1515ed5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:59:08,735 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ff8009e463e,46289,-1 for getting cluster id 2024-11-25T12:59:08,735 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:59:08,736 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb972fd5-cdcb-422b-acfe-41fc80d301d9' 2024-11-25T12:59:08,736 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:59:08,736 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb972fd5-cdcb-422b-acfe-41fc80d301d9" 2024-11-25T12:59:08,736 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22d3eb46, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:59:08,736 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ff8009e463e,46289,-1] 2024-11-25T12:59:08,736 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:59:08,736 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:59:08,737 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59330, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:59:08,738 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@119d007f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:59:08,738 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d10c554, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:59:08,738 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ff8009e463e,46289,-1 for getting cluster id 2024-11-25T12:59:08,738 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:59:08,739 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb972fd5-cdcb-422b-acfe-41fc80d301d9' 2024-11-25T12:59:08,739 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:59:08,739 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb972fd5-cdcb-422b-acfe-41fc80d301d9" 2024-11-25T12:59:08,739 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e99e41b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:59:08,739 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ff8009e463e,46289,-1] 2024-11-25T12:59:08,739 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:59:08,739 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:59:08,740 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59338, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:59:08,741 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fdb4069, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:59:08,741 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26b7b919, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:59:08,741 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ff8009e463e,46289,-1 for getting cluster id 2024-11-25T12:59:08,741 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:59:08,742 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb972fd5-cdcb-422b-acfe-41fc80d301d9' 2024-11-25T12:59:08,742 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:59:08,742 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb972fd5-cdcb-422b-acfe-41fc80d301d9" 2024-11-25T12:59:08,742 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33e49748, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:59:08,742 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ff8009e463e,46289,-1] 2024-11-25T12:59:08,742 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:59:08,743 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:59:08,743 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59342, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:59:08,744 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1227548, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:59:08,744 DEBUG [Thread-1488 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:59:08,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,35905,1732539449246, seqNum=-1] 2024-11-25T12:59:08,745 DEBUG [Thread-1490 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:59:08,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:59:08,745 DEBUG [Thread-1477 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:59:08,745 DEBUG [Thread-1483 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:59:08,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,35905,1732539449246, seqNum=-1] 2024-11-25T12:59:08,746 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46464, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:59:08,746 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,35905,1732539449246, seqNum=-1] 2024-11-25T12:59:08,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:59:08,746 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:59:08,746 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,35905,1732539449246, seqNum=-1] 2024-11-25T12:59:08,746 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:59:08,747 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46494, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:59:08,748 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46484, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:59:08,748 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:59:08,748 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46474, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:59:08,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5] 2024-11-25T12:59:08,749 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5] 2024-11-25T12:59:08,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5] 2024-11-25T12:59:08,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] procedure2.ProcedureExecutor(1139): Stored pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees 2024-11-25T12:59:08,750 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5] 2024-11-25T12:59:08,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-25T12:59:08,751 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:59:08,751 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:59:08,752 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:59:08,755 DEBUG [Thread-1496 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:59:08,756 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,35905,1732539449246, seqNum=-1] 2024-11-25T12:59:08,756 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:59:08,758 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46504, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:59:08,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:08,759 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:59:08,759 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing de45cbd8ecc09a85a2c9aae45053021b 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-25T12:59:08,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5] 2024-11-25T12:59:08,763 DEBUG [Thread-1494 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:59:08,765 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,35905,1732539449246, seqNum=-1] 2024-11-25T12:59:08,765 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:59:08,766 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46510, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:59:08,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5] 2024-11-25T12:59:08,768 DEBUG [Thread-1485 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:59:08,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,35905,1732539449246, seqNum=-1] 2024-11-25T12:59:08,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:59:08,770 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46522, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:59:08,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5] 2024-11-25T12:59:08,787 DEBUG [Thread-1481 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:59:08,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,35905,1732539449246, seqNum=-1] 2024-11-25T12:59:08,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:59:08,789 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46538, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:59:08,791 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5] 2024-11-25T12:59:08,799 DEBUG [Thread-1479 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:59:08,800 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,35905,1732539449246, seqNum=-1] 2024-11-25T12:59:08,801 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:59:08,823 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46542, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:59:08,823 DEBUG [Thread-1492 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:59:08,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5] 2024-11-25T12:59:08,826 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,35905,1732539449246, seqNum=-1] 2024-11-25T12:59:08,827 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:59:08,828 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46548, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:59:08,831 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5] 2024-11-25T12:59:08,833 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411250a6da13d459a41b4a23570b3037f6621_de45cbd8ecc09a85a2c9aae45053021b is 50, key is test_row_1/A:col10/1732539548758/Put/seqid=0 2024-11-25T12:59:08,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742146_1322 (size=14594) 2024-11-25T12:59:08,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:08,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46538 deadline: 1732539608827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:08,851 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:08,851 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:08,851 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:08,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:08,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46484 deadline: 1732539608837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:08,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:08,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:08,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:08,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-25T12:59:08,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:08,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46522 deadline: 1732539608850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:08,861 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:08,861 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:08,861 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:08,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:08,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46474 deadline: 1732539608851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:08,861 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:08,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:08,861 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:08,862 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:08,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 2 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46542 deadline: 1732539608852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:08,862 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:08,863 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:08,863 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:08,912 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-25T12:59:08,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:08,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. as already flushing 2024-11-25T12:59:08,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:08,913 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:08,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:08,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:08,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:08,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46538 deadline: 1732539608962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:08,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:08,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:08,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:08,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:08,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46522 deadline: 1732539608972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:08,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:08,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:08,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:08,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:08,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46474 deadline: 1732539608972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:08,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:08,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46484 deadline: 1732539608975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:08,982 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:08,982 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:08,982 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:08,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:08,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:08,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:08,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:08,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46542 deadline: 1732539608975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:08,984 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:08,984 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:08,984 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:09,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-25T12:59:09,074 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-25T12:59:09,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:09,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. as already flushing 2024-11-25T12:59:09,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:09,074 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:09,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:09,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:09,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:09,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46538 deadline: 1732539609170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:09,175 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:09,175 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:09,175 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:09,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:09,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:09,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46522 deadline: 1732539609192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:09,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46542 deadline: 1732539609192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:09,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:09,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:09,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:09,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:09,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46474 deadline: 1732539609192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:09,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:09,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46484 deadline: 1732539609192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:09,201 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:09,201 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:09,201 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:09,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:09,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:09,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:09,206 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:09,206 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:09,206 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:09,232 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-25T12:59:09,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:09,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. as already flushing 2024-11-25T12:59:09,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:09,232 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:09,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:09,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:09,242 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:09,247 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411250a6da13d459a41b4a23570b3037f6621_de45cbd8ecc09a85a2c9aae45053021b to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411250a6da13d459a41b4a23570b3037f6621_de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:09,249 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/6ba8a71ff08d4cd08b2f44d8330fd808, store: [table=TestAcidGuarantees family=A region=de45cbd8ecc09a85a2c9aae45053021b] 2024-11-25T12:59:09,250 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/6ba8a71ff08d4cd08b2f44d8330fd808 is 175, key is test_row_1/A:col10/1732539548758/Put/seqid=0 2024-11-25T12:59:09,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742147_1323 (size=39545) 2024-11-25T12:59:09,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-25T12:59:09,385 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-25T12:59:09,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:09,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. as already flushing 2024-11-25T12:59:09,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:09,386 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:09,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:09,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:09,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:09,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46538 deadline: 1732539609480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:09,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:09,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:09,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:09,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:09,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46522 deadline: 1732539609510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:09,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:09,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:09,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:09,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:09,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46474 deadline: 1732539609510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:09,520 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:09,520 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:09,520 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:09,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:09,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46542 deadline: 1732539609511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:09,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:09,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46484 deadline: 1732539609511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:09,521 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:09,521 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:09,521 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:09,523 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:09,523 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:09,523 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:09,539 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-25T12:59:09,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:09,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. as already flushing 2024-11-25T12:59:09,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:09,541 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:09,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:09,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:09,684 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/6ba8a71ff08d4cd08b2f44d8330fd808 2024-11-25T12:59:09,695 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-25T12:59:09,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:09,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. as already flushing 2024-11-25T12:59:09,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:09,695 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:09,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:09,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:09,710 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/B/dbfc859d33714e3a8c25f1e91081474d is 50, key is test_row_1/B:col10/1732539548758/Put/seqid=0 2024-11-25T12:59:09,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742148_1324 (size=9657) 2024-11-25T12:59:09,729 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/B/dbfc859d33714e3a8c25f1e91081474d 2024-11-25T12:59:09,763 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/C/26df2c6ab7a741e2ad34b54a06238814 is 50, key is test_row_1/C:col10/1732539548758/Put/seqid=0 2024-11-25T12:59:09,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742149_1325 (size=9657) 2024-11-25T12:59:09,850 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-25T12:59:09,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:09,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. as already flushing 2024-11-25T12:59:09,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:09,850 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:09,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:09,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:09,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-25T12:59:10,003 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-25T12:59:10,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:10,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. as already flushing 2024-11-25T12:59:10,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:10,003 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:10,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:10,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:10,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46538 deadline: 1732539610000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:10,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:10,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:10,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:10,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:10,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:10,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46474 deadline: 1732539610030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:10,032 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:10,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:10,032 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:10,032 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:10,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46522 deadline: 1732539610030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:10,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:10,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46484 deadline: 1732539610030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:10,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:10,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:10,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:10,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:10,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:10,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46542 deadline: 1732539610030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:10,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:10,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:10,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:10,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:10,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:10,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-25T12:59:10,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:10,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. as already flushing 2024-11-25T12:59:10,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:10,162 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:10,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:10,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:10,192 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/C/26df2c6ab7a741e2ad34b54a06238814 2024-11-25T12:59:10,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/6ba8a71ff08d4cd08b2f44d8330fd808 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/6ba8a71ff08d4cd08b2f44d8330fd808 2024-11-25T12:59:10,214 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/6ba8a71ff08d4cd08b2f44d8330fd808, entries=200, sequenceid=15, filesize=38.6 K 2024-11-25T12:59:10,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/B/dbfc859d33714e3a8c25f1e91081474d as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/dbfc859d33714e3a8c25f1e91081474d 2024-11-25T12:59:10,219 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/dbfc859d33714e3a8c25f1e91081474d, entries=100, sequenceid=15, filesize=9.4 K 2024-11-25T12:59:10,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/C/26df2c6ab7a741e2ad34b54a06238814 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/26df2c6ab7a741e2ad34b54a06238814 2024-11-25T12:59:10,226 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/26df2c6ab7a741e2ad34b54a06238814, entries=100, sequenceid=15, filesize=9.4 K 2024-11-25T12:59:10,227 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=161.02 KB/164880 for de45cbd8ecc09a85a2c9aae45053021b in 1468ms, sequenceid=15, compaction requested=false 2024-11-25T12:59:10,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for de45cbd8ecc09a85a2c9aae45053021b: 2024-11-25T12:59:10,322 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-25T12:59:10,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:10,322 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2902): Flushing de45cbd8ecc09a85a2c9aae45053021b 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-25T12:59:10,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125c1f6bf90f45c4c08adda9bff93940572_de45cbd8ecc09a85a2c9aae45053021b is 50, key is test_row_0/A:col10/1732539548822/Put/seqid=0 2024-11-25T12:59:10,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742150_1326 (size=12154) 2024-11-25T12:59:10,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,365 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125c1f6bf90f45c4c08adda9bff93940572_de45cbd8ecc09a85a2c9aae45053021b to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125c1f6bf90f45c4c08adda9bff93940572_de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:10,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/24af528763924d2293e8f9d80e391407, store: [table=TestAcidGuarantees family=A region=de45cbd8ecc09a85a2c9aae45053021b] 2024-11-25T12:59:10,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/24af528763924d2293e8f9d80e391407 is 175, key is test_row_0/A:col10/1732539548822/Put/seqid=0 2024-11-25T12:59:10,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742151_1327 (size=30955) 2024-11-25T12:59:10,395 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=42, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/24af528763924d2293e8f9d80e391407 2024-11-25T12:59:10,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/B/bc614b2ddf3e4888a43bc57d5c99b1bb is 50, key is test_row_0/B:col10/1732539548822/Put/seqid=0 2024-11-25T12:59:10,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742152_1328 (size=12001) 2024-11-25T12:59:10,430 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/B/bc614b2ddf3e4888a43bc57d5c99b1bb 2024-11-25T12:59:10,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/C/075b4de41cba48a59dc9e7d580f7d67d is 50, key is test_row_0/C:col10/1732539548822/Put/seqid=0 2024-11-25T12:59:10,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742153_1329 (size=12001) 2024-11-25T12:59:10,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-25T12:59:10,922 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/C/075b4de41cba48a59dc9e7d580f7d67d 2024-11-25T12:59:10,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/24af528763924d2293e8f9d80e391407 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/24af528763924d2293e8f9d80e391407 2024-11-25T12:59:10,938 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/24af528763924d2293e8f9d80e391407, entries=150, sequenceid=42, filesize=30.2 K 2024-11-25T12:59:10,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/B/bc614b2ddf3e4888a43bc57d5c99b1bb as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/bc614b2ddf3e4888a43bc57d5c99b1bb 2024-11-25T12:59:10,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,944 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/bc614b2ddf3e4888a43bc57d5c99b1bb, entries=150, sequenceid=42, filesize=11.7 K 2024-11-25T12:59:10,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/C/075b4de41cba48a59dc9e7d580f7d67d as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/075b4de41cba48a59dc9e7d580f7d67d 2024-11-25T12:59:10,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,949 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/075b4de41cba48a59dc9e7d580f7d67d, entries=150, sequenceid=42, filesize=11.7 K 2024-11-25T12:59:10,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,950 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=0 B/0 for de45cbd8ecc09a85a2c9aae45053021b in 628ms, sequenceid=42, compaction requested=false 2024-11-25T12:59:10,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2603): Flush status journal for de45cbd8ecc09a85a2c9aae45053021b: 2024-11-25T12:59:10,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:10,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=90 2024-11-25T12:59:10,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4169): Remote procedure done, pid=90 2024-11-25T12:59:10,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,953 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-25T12:59:10,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,953 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2010 sec 2024-11-25T12:59:10,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,955 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees in 2.2060 sec 2024-11-25T12:59:10,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:10,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,203 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:59:11,203 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing de45cbd8ecc09a85a2c9aae45053021b 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-25T12:59:11,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:11,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,218 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125141d2322abcb4cdcbd73937ad41ea8eb_de45cbd8ecc09a85a2c9aae45053021b is 50, key is test_row_0/A:col10/1732539551193/Put/seqid=0 2024-11-25T12:59:11,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,240 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T12:59:11,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742155_1331 (size=24358) 2024-11-25T12:59:11,253 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:11,259 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125141d2322abcb4cdcbd73937ad41ea8eb_de45cbd8ecc09a85a2c9aae45053021b to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125141d2322abcb4cdcbd73937ad41ea8eb_de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:11,260 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/92d22b71473d4daf955c71692c758a90, store: [table=TestAcidGuarantees family=A region=de45cbd8ecc09a85a2c9aae45053021b] 2024-11-25T12:59:11,271 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/92d22b71473d4daf955c71692c758a90 is 175, key is test_row_0/A:col10/1732539551193/Put/seqid=0 2024-11-25T12:59:11,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742154_1330 (size=73995) 2024-11-25T12:59:11,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:11,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46474 deadline: 1732539611358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:11,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:11,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46538 deadline: 1732539611358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:11,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:11,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46522 deadline: 1732539611360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:11,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:11,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46542 deadline: 1732539611363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:11,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:11,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:11,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:11,382 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:11,382 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:11,382 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:11,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:11,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:11,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:11,383 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:11,383 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:11,383 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:11,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:11,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46484 deadline: 1732539611382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:11,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:11,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:11,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:11,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:11,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46542 deadline: 1732539611492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:11,506 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:11,506 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:11,506 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:11,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:11,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46522 deadline: 1732539611492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:11,506 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:11,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:11,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:11,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:11,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46474 deadline: 1732539611492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:11,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:11,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46538 deadline: 1732539611492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:11,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:11,512 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:11,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:11,512 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:11,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:11,512 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:11,524 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:11,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46484 deadline: 1732539611509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:11,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:11,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:11,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:11,680 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=53, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/92d22b71473d4daf955c71692c758a90 2024-11-25T12:59:11,714 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/B/1e6e10ae809c4a87885da5c22a214ff2 is 50, key is test_row_0/B:col10/1732539551193/Put/seqid=0 2024-11-25T12:59:11,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:11,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46542 deadline: 1732539611714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:11,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:11,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46522 deadline: 1732539611714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:11,726 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:11,726 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:11,727 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:11,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:11,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:11,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:11,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:11,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46538 deadline: 1732539611721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:11,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:11,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46474 deadline: 1732539611721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:11,730 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:11,730 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:11,730 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:11,730 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:11,730 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:11,730 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:11,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:11,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46484 deadline: 1732539611731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:11,742 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:11,742 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:11,742 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:11,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742156_1332 (size=12001) 2024-11-25T12:59:12,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:12,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46542 deadline: 1732539612030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:12,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:12,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46522 deadline: 1732539612040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:12,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:12,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:12,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:12,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:12,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46474 deadline: 1732539612040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:12,050 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:12,050 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:12,050 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:12,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:12,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:12,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:12,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:12,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46538 deadline: 1732539612041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:12,050 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:12,050 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:12,050 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:12,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:12,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46484 deadline: 1732539612050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:12,069 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:12,069 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:12,069 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:12,160 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/B/1e6e10ae809c4a87885da5c22a214ff2 2024-11-25T12:59:12,183 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/C/2b66783f288e4f9098d11afc71c24698 is 50, key is test_row_0/C:col10/1732539551193/Put/seqid=0 2024-11-25T12:59:12,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742157_1333 (size=12001) 2024-11-25T12:59:12,244 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/C/2b66783f288e4f9098d11afc71c24698 2024-11-25T12:59:12,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/92d22b71473d4daf955c71692c758a90 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/92d22b71473d4daf955c71692c758a90 2024-11-25T12:59:12,254 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/92d22b71473d4daf955c71692c758a90, entries=400, sequenceid=53, filesize=72.3 K 2024-11-25T12:59:12,254 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/B/1e6e10ae809c4a87885da5c22a214ff2 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/1e6e10ae809c4a87885da5c22a214ff2 2024-11-25T12:59:12,259 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/1e6e10ae809c4a87885da5c22a214ff2, entries=150, sequenceid=53, filesize=11.7 K 2024-11-25T12:59:12,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/C/2b66783f288e4f9098d11afc71c24698 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/2b66783f288e4f9098d11afc71c24698 2024-11-25T12:59:12,267 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/2b66783f288e4f9098d11afc71c24698, entries=150, sequenceid=53, filesize=11.7 K 2024-11-25T12:59:12,270 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for de45cbd8ecc09a85a2c9aae45053021b in 1067ms, sequenceid=53, compaction requested=true 2024-11-25T12:59:12,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for de45cbd8ecc09a85a2c9aae45053021b: 2024-11-25T12:59:12,270 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:59:12,271 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 144495 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:59:12,271 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1541): de45cbd8ecc09a85a2c9aae45053021b/A is initiating minor compaction (all files) 2024-11-25T12:59:12,271 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de45cbd8ecc09a85a2c9aae45053021b/A in TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:12,272 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/6ba8a71ff08d4cd08b2f44d8330fd808, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/24af528763924d2293e8f9d80e391407, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/92d22b71473d4daf955c71692c758a90] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp, totalSize=141.1 K 2024-11-25T12:59:12,272 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:12,272 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. files: [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/6ba8a71ff08d4cd08b2f44d8330fd808, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/24af528763924d2293e8f9d80e391407, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/92d22b71473d4daf955c71692c758a90] 2024-11-25T12:59:12,272 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ba8a71ff08d4cd08b2f44d8330fd808, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732539548754 2024-11-25T12:59:12,273 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24af528763924d2293e8f9d80e391407, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1732539548822 2024-11-25T12:59:12,273 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92d22b71473d4daf955c71692c758a90, keycount=400, bloomtype=ROW, size=72.3 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732539551138 2024-11-25T12:59:12,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de45cbd8ecc09a85a2c9aae45053021b:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:59:12,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:59:12,296 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:59:12,297 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:59:12,297 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): de45cbd8ecc09a85a2c9aae45053021b/B is initiating minor compaction (all files) 2024-11-25T12:59:12,297 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de45cbd8ecc09a85a2c9aae45053021b/B in TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:12,297 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/dbfc859d33714e3a8c25f1e91081474d, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/bc614b2ddf3e4888a43bc57d5c99b1bb, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/1e6e10ae809c4a87885da5c22a214ff2] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp, totalSize=32.9 K 2024-11-25T12:59:12,297 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting dbfc859d33714e3a8c25f1e91081474d, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732539548758 2024-11-25T12:59:12,298 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting bc614b2ddf3e4888a43bc57d5c99b1bb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1732539548822 2024-11-25T12:59:12,298 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e6e10ae809c4a87885da5c22a214ff2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732539551146 2024-11-25T12:59:12,300 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=de45cbd8ecc09a85a2c9aae45053021b] 2024-11-25T12:59:12,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de45cbd8ecc09a85a2c9aae45053021b:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:59:12,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:59:12,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de45cbd8ecc09a85a2c9aae45053021b:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:59:12,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:59:12,320 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241125072c80c1135a43c4a8ab26fed8b05230_de45cbd8ecc09a85a2c9aae45053021b store=[table=TestAcidGuarantees family=A region=de45cbd8ecc09a85a2c9aae45053021b] 2024-11-25T12:59:12,323 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241125072c80c1135a43c4a8ab26fed8b05230_de45cbd8ecc09a85a2c9aae45053021b, store=[table=TestAcidGuarantees family=A region=de45cbd8ecc09a85a2c9aae45053021b] 2024-11-25T12:59:12,324 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125072c80c1135a43c4a8ab26fed8b05230_de45cbd8ecc09a85a2c9aae45053021b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=de45cbd8ecc09a85a2c9aae45053021b] 2024-11-25T12:59:12,334 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): de45cbd8ecc09a85a2c9aae45053021b#B#compaction#283 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:59:12,335 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/B/668e0480eff441cb8abc21b4032cba27 is 50, key is test_row_0/B:col10/1732539551193/Put/seqid=0 2024-11-25T12:59:12,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742158_1334 (size=4469) 2024-11-25T12:59:12,400 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): de45cbd8ecc09a85a2c9aae45053021b#A#compaction#282 average throughput is 0.24 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:59:12,401 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/b1e05ed9177e4b87972f9ae3f0c069d1 is 175, key is test_row_0/A:col10/1732539551193/Put/seqid=0 2024-11-25T12:59:12,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742159_1335 (size=12104) 2024-11-25T12:59:12,415 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/B/668e0480eff441cb8abc21b4032cba27 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/668e0480eff441cb8abc21b4032cba27 2024-11-25T12:59:12,422 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de45cbd8ecc09a85a2c9aae45053021b/B of de45cbd8ecc09a85a2c9aae45053021b into 668e0480eff441cb8abc21b4032cba27(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:59:12,422 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de45cbd8ecc09a85a2c9aae45053021b: 2024-11-25T12:59:12,422 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., storeName=de45cbd8ecc09a85a2c9aae45053021b/B, priority=13, startTime=1732539552295; duration=0sec 2024-11-25T12:59:12,423 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:59:12,423 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de45cbd8ecc09a85a2c9aae45053021b:B 2024-11-25T12:59:12,423 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:59:12,424 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:59:12,424 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): de45cbd8ecc09a85a2c9aae45053021b/C is initiating minor compaction (all files) 2024-11-25T12:59:12,424 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de45cbd8ecc09a85a2c9aae45053021b/C in TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:12,424 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/26df2c6ab7a741e2ad34b54a06238814, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/075b4de41cba48a59dc9e7d580f7d67d, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/2b66783f288e4f9098d11afc71c24698] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp, totalSize=32.9 K 2024-11-25T12:59:12,424 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 26df2c6ab7a741e2ad34b54a06238814, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732539548758 2024-11-25T12:59:12,425 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 075b4de41cba48a59dc9e7d580f7d67d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1732539548822 2024-11-25T12:59:12,425 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b66783f288e4f9098d11afc71c24698, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732539551146 2024-11-25T12:59:12,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742160_1336 (size=31058) 2024-11-25T12:59:12,466 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): de45cbd8ecc09a85a2c9aae45053021b#C#compaction#284 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:59:12,467 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/C/0ddd0ca5d2c040e68b48a0d397fb9cea is 50, key is test_row_0/C:col10/1732539551193/Put/seqid=0 2024-11-25T12:59:12,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742161_1337 (size=12104) 2024-11-25T12:59:12,526 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/C/0ddd0ca5d2c040e68b48a0d397fb9cea as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/0ddd0ca5d2c040e68b48a0d397fb9cea 2024-11-25T12:59:12,531 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de45cbd8ecc09a85a2c9aae45053021b/C of de45cbd8ecc09a85a2c9aae45053021b into 0ddd0ca5d2c040e68b48a0d397fb9cea(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:59:12,531 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de45cbd8ecc09a85a2c9aae45053021b: 2024-11-25T12:59:12,531 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., storeName=de45cbd8ecc09a85a2c9aae45053021b/C, priority=13, startTime=1732539552304; duration=0sec 2024-11-25T12:59:12,531 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:59:12,532 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de45cbd8ecc09a85a2c9aae45053021b:C 2024-11-25T12:59:12,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:12,569 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:59:12,570 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing de45cbd8ecc09a85a2c9aae45053021b 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-25T12:59:12,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:12,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46522 deadline: 1732539612584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:12,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:12,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:12,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:12,597 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411250c3555227eae4d7ebb928f492d4aa90a_de45cbd8ecc09a85a2c9aae45053021b is 50, key is test_row_0/A:col10/1732539552569/Put/seqid=0 2024-11-25T12:59:12,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:12,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46474 deadline: 1732539612591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:12,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:12,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:12,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:12,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742162_1338 (size=17034) 2024-11-25T12:59:12,616 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:12,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:12,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46484 deadline: 1732539612596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:12,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:12,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:12,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:12,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:12,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46542 deadline: 1732539612596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:12,617 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:12,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:12,617 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:12,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46538 deadline: 1732539612598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:12,617 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:12,617 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:12,617 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:12,617 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:12,622 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411250c3555227eae4d7ebb928f492d4aa90a_de45cbd8ecc09a85a2c9aae45053021b to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411250c3555227eae4d7ebb928f492d4aa90a_de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:12,623 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/94baa5c4494549cf87acc14ea1a33dc9, store: [table=TestAcidGuarantees family=A region=de45cbd8ecc09a85a2c9aae45053021b] 2024-11-25T12:59:12,624 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/94baa5c4494549cf87acc14ea1a33dc9 is 175, key is test_row_0/A:col10/1732539552569/Put/seqid=0 2024-11-25T12:59:12,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742163_1339 (size=48139) 2024-11-25T12:59:12,644 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=82, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/94baa5c4494549cf87acc14ea1a33dc9 2024-11-25T12:59:12,666 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/B/0db34574bfab49e3978f2f7d0120cfcc is 50, key is test_row_0/B:col10/1732539552569/Put/seqid=0 2024-11-25T12:59:12,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742164_1340 (size=12001) 2024-11-25T12:59:12,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:12,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46522 deadline: 1732539612700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:12,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:12,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:12,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:12,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:12,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46474 deadline: 1732539612710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:12,728 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:12,728 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:12,728 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:12,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:12,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46484 deadline: 1732539612727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:12,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:12,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46542 deadline: 1732539612727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:12,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:12,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46538 deadline: 1732539612727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:12,739 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:12,739 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:12,739 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:12,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:12,746 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:12,746 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:12,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:12,746 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:12,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:12,874 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/b1e05ed9177e4b87972f9ae3f0c069d1 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/b1e05ed9177e4b87972f9ae3f0c069d1 2024-11-25T12:59:12,886 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de45cbd8ecc09a85a2c9aae45053021b/A of de45cbd8ecc09a85a2c9aae45053021b into b1e05ed9177e4b87972f9ae3f0c069d1(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:59:12,886 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de45cbd8ecc09a85a2c9aae45053021b: 2024-11-25T12:59:12,887 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., storeName=de45cbd8ecc09a85a2c9aae45053021b/A, priority=13, startTime=1732539552270; duration=0sec 2024-11-25T12:59:12,887 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:59:12,887 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de45cbd8ecc09a85a2c9aae45053021b:A 2024-11-25T12:59:12,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-25T12:59:12,910 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:59:12,912 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:59:12,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees 2024-11-25T12:59:12,914 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:59:12,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-25T12:59:12,915 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:59:12,915 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:59:12,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:12,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46522 deadline: 1732539612930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:12,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:12,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46474 deadline: 1732539612930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:12,941 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:12,941 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:12,941 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:12,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:12,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:12,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:12,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:12,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46484 deadline: 1732539612952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:12,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:12,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:12,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:12,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46538 deadline: 1732539612952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:12,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:12,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:12,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:12,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:12,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:12,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46542 deadline: 1732539612952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:12,975 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:12,975 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:12,975 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:13,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-25T12:59:13,068 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-25T12:59:13,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:13,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. as already flushing 2024-11-25T12:59:13,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:13,068 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:13,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:13,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:13,112 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/B/0db34574bfab49e3978f2f7d0120cfcc 2024-11-25T12:59:13,145 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/C/d810f965a4044ebfa0e724befc983c29 is 50, key is test_row_0/C:col10/1732539552569/Put/seqid=0 2024-11-25T12:59:13,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742165_1341 (size=12001) 2024-11-25T12:59:13,226 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/C/d810f965a4044ebfa0e724befc983c29 2024-11-25T12:59:13,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/94baa5c4494549cf87acc14ea1a33dc9 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/94baa5c4494549cf87acc14ea1a33dc9 2024-11-25T12:59:13,236 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-25T12:59:13,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:13,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. as already flushing 2024-11-25T12:59:13,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:13,237 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:13,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:13,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:13,240 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/94baa5c4494549cf87acc14ea1a33dc9, entries=250, sequenceid=82, filesize=47.0 K 2024-11-25T12:59:13,248 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/B/0db34574bfab49e3978f2f7d0120cfcc as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/0db34574bfab49e3978f2f7d0120cfcc 2024-11-25T12:59:13,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:13,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46474 deadline: 1732539613250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:13,260 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:13,260 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:13,260 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:13,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:13,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46522 deadline: 1732539613251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:13,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:13,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:13,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:13,264 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/0db34574bfab49e3978f2f7d0120cfcc, entries=150, sequenceid=82, filesize=11.7 K 2024-11-25T12:59:13,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/C/d810f965a4044ebfa0e724befc983c29 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/d810f965a4044ebfa0e724befc983c29 2024-11-25T12:59:13,269 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/d810f965a4044ebfa0e724befc983c29, entries=150, sequenceid=82, filesize=11.7 K 2024-11-25T12:59:13,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-25T12:59:13,271 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=46.96 KB/48090 for de45cbd8ecc09a85a2c9aae45053021b in 702ms, sequenceid=82, compaction requested=false 2024-11-25T12:59:13,271 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for de45cbd8ecc09a85a2c9aae45053021b: 2024-11-25T12:59:13,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:13,289 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:59:13,289 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing de45cbd8ecc09a85a2c9aae45053021b 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-25T12:59:13,322 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112539ec670bb3a84a18aaf0e26e1eb51abe_de45cbd8ecc09a85a2c9aae45053021b is 50, key is test_row_0/A:col10/1732539553287/Put/seqid=0 2024-11-25T12:59:13,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742166_1342 (size=17034) 2024-11-25T12:59:13,367 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:13,389 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112539ec670bb3a84a18aaf0e26e1eb51abe_de45cbd8ecc09a85a2c9aae45053021b to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112539ec670bb3a84a18aaf0e26e1eb51abe_de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:13,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-25T12:59:13,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:13,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. as already flushing 2024-11-25T12:59:13,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:13,393 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:13,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:13,394 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/97d59372c3164d30936aad8644d60849, store: [table=TestAcidGuarantees family=A region=de45cbd8ecc09a85a2c9aae45053021b] 2024-11-25T12:59:13,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:13,394 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/97d59372c3164d30936aad8644d60849 is 175, key is test_row_0/A:col10/1732539553287/Put/seqid=0 2024-11-25T12:59:13,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742167_1343 (size=48139) 2024-11-25T12:59:13,401 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=96, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/97d59372c3164d30936aad8644d60849 2024-11-25T12:59:13,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:13,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46542 deadline: 1732539613415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:13,427 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:13,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:13,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:13,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:13,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46538 deadline: 1732539613418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:13,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:13,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:13,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:13,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:13,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46484 deadline: 1732539613423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:13,432 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/B/73817ec81d5f4eb19f2392140fb9f60b is 50, key is test_row_0/B:col10/1732539553287/Put/seqid=0 2024-11-25T12:59:13,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:13,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:13,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:13,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742168_1344 (size=12001) 2024-11-25T12:59:13,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:13,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46542 deadline: 1732539613530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:13,540 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:13,540 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:13,540 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:13,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:13,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46538 deadline: 1732539613536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:13,542 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:13,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:13,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:13,550 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-25T12:59:13,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:13,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. as already flushing 2024-11-25T12:59:13,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:13,551 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:13,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:13,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:13,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:13,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46484 deadline: 1732539613540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:13,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:13,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:13,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:13,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-25T12:59:13,704 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-25T12:59:13,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:13,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. as already flushing 2024-11-25T12:59:13,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:13,705 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:13,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:13,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:13,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:13,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46542 deadline: 1732539613753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:13,763 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:13,763 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:13,763 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:13,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:13,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46538 deadline: 1732539613755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:13,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:13,772 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:13,772 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:13,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:13,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46484 deadline: 1732539613772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:13,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:13,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:13,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:13,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:13,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46522 deadline: 1732539613772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:13,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:13,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:13,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:13,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:13,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46474 deadline: 1732539613775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:13,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:13,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:13,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:13,861 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-25T12:59:13,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:13,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. as already flushing 2024-11-25T12:59:13,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:13,862 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:13,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:13,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:13,880 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/B/73817ec81d5f4eb19f2392140fb9f60b 2024-11-25T12:59:13,888 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/C/e9dfe42e7c76473282927aa94baffaee is 50, key is test_row_0/C:col10/1732539553287/Put/seqid=0 2024-11-25T12:59:13,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742169_1345 (size=12001) 2024-11-25T12:59:14,015 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-25T12:59:14,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:14,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. as already flushing 2024-11-25T12:59:14,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:14,016 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:14,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:14,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:14,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:14,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46542 deadline: 1732539614071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:14,075 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:14,075 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:14,075 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:14,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:14,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46538 deadline: 1732539614080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:14,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:14,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:14,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:14,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:14,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46484 deadline: 1732539614080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:14,089 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:14,089 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:14,089 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:14,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-25T12:59:14,170 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-25T12:59:14,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:14,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. as already flushing 2024-11-25T12:59:14,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:14,171 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:14,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:14,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:14,319 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/C/e9dfe42e7c76473282927aa94baffaee 2024-11-25T12:59:14,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-25T12:59:14,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:14,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. as already flushing 2024-11-25T12:59:14,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:14,324 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:14,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:14,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:14,336 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/97d59372c3164d30936aad8644d60849 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/97d59372c3164d30936aad8644d60849 2024-11-25T12:59:14,339 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/97d59372c3164d30936aad8644d60849, entries=250, sequenceid=96, filesize=47.0 K 2024-11-25T12:59:14,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/B/73817ec81d5f4eb19f2392140fb9f60b as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/73817ec81d5f4eb19f2392140fb9f60b 2024-11-25T12:59:14,344 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/73817ec81d5f4eb19f2392140fb9f60b, entries=150, sequenceid=96, filesize=11.7 K 2024-11-25T12:59:14,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/C/e9dfe42e7c76473282927aa94baffaee as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/e9dfe42e7c76473282927aa94baffaee 2024-11-25T12:59:14,349 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/e9dfe42e7c76473282927aa94baffaee, entries=150, sequenceid=96, filesize=11.7 K 2024-11-25T12:59:14,355 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for de45cbd8ecc09a85a2c9aae45053021b in 1066ms, sequenceid=96, compaction requested=true 2024-11-25T12:59:14,356 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for de45cbd8ecc09a85a2c9aae45053021b: 2024-11-25T12:59:14,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de45cbd8ecc09a85a2c9aae45053021b:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:59:14,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:59:14,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de45cbd8ecc09a85a2c9aae45053021b:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:59:14,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-25T12:59:14,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de45cbd8ecc09a85a2c9aae45053021b:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:59:14,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-25T12:59:14,356 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:59:14,356 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:59:14,357 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:59:14,357 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): de45cbd8ecc09a85a2c9aae45053021b/C is initiating minor compaction (all files) 2024-11-25T12:59:14,357 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de45cbd8ecc09a85a2c9aae45053021b/C in TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:14,357 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/0ddd0ca5d2c040e68b48a0d397fb9cea, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/d810f965a4044ebfa0e724befc983c29, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/e9dfe42e7c76473282927aa94baffaee] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp, totalSize=35.3 K 2024-11-25T12:59:14,358 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 127336 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:59:14,358 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1541): de45cbd8ecc09a85a2c9aae45053021b/A is initiating minor compaction (all files) 2024-11-25T12:59:14,358 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de45cbd8ecc09a85a2c9aae45053021b/A in TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:14,358 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/b1e05ed9177e4b87972f9ae3f0c069d1, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/94baa5c4494549cf87acc14ea1a33dc9, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/97d59372c3164d30936aad8644d60849] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp, totalSize=124.4 K 2024-11-25T12:59:14,358 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:14,358 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. files: [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/b1e05ed9177e4b87972f9ae3f0c069d1, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/94baa5c4494549cf87acc14ea1a33dc9, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/97d59372c3164d30936aad8644d60849] 2024-11-25T12:59:14,359 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ddd0ca5d2c040e68b48a0d397fb9cea, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732539551146 2024-11-25T12:59:14,359 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1e05ed9177e4b87972f9ae3f0c069d1, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732539551146 2024-11-25T12:59:14,359 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting d810f965a4044ebfa0e724befc983c29, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1732539551359 2024-11-25T12:59:14,359 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94baa5c4494549cf87acc14ea1a33dc9, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1732539551330 2024-11-25T12:59:14,359 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting e9dfe42e7c76473282927aa94baffaee, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732539552578 2024-11-25T12:59:14,360 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97d59372c3164d30936aad8644d60849, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732539552578 2024-11-25T12:59:14,377 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): de45cbd8ecc09a85a2c9aae45053021b#C#compaction#291 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:59:14,378 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/C/dc5a19e2d9fa4ced956635a857081496 is 50, key is test_row_0/C:col10/1732539553287/Put/seqid=0 2024-11-25T12:59:14,389 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=de45cbd8ecc09a85a2c9aae45053021b] 2024-11-25T12:59:14,412 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241125ce5900b2ea29443a82b4d8ddd5414c61_de45cbd8ecc09a85a2c9aae45053021b store=[table=TestAcidGuarantees family=A region=de45cbd8ecc09a85a2c9aae45053021b] 2024-11-25T12:59:14,414 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241125ce5900b2ea29443a82b4d8ddd5414c61_de45cbd8ecc09a85a2c9aae45053021b, store=[table=TestAcidGuarantees family=A region=de45cbd8ecc09a85a2c9aae45053021b] 2024-11-25T12:59:14,414 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125ce5900b2ea29443a82b4d8ddd5414c61_de45cbd8ecc09a85a2c9aae45053021b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=de45cbd8ecc09a85a2c9aae45053021b] 2024-11-25T12:59:14,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742170_1346 (size=12207) 2024-11-25T12:59:14,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742171_1347 (size=4469) 2024-11-25T12:59:14,456 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/C/dc5a19e2d9fa4ced956635a857081496 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/dc5a19e2d9fa4ced956635a857081496 2024-11-25T12:59:14,460 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): de45cbd8ecc09a85a2c9aae45053021b#A#compaction#292 average throughput is 0.34 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:59:14,461 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/fa0837cd25d244b89fdbe325a806773e is 175, key is test_row_0/A:col10/1732539553287/Put/seqid=0 2024-11-25T12:59:14,475 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de45cbd8ecc09a85a2c9aae45053021b/C of de45cbd8ecc09a85a2c9aae45053021b into dc5a19e2d9fa4ced956635a857081496(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:59:14,475 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de45cbd8ecc09a85a2c9aae45053021b: 2024-11-25T12:59:14,475 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., storeName=de45cbd8ecc09a85a2c9aae45053021b/C, priority=13, startTime=1732539554356; duration=0sec 2024-11-25T12:59:14,475 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:59:14,475 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de45cbd8ecc09a85a2c9aae45053021b:C 2024-11-25T12:59:14,476 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:59:14,477 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:59:14,477 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): de45cbd8ecc09a85a2c9aae45053021b/B is initiating minor compaction (all files) 2024-11-25T12:59:14,477 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de45cbd8ecc09a85a2c9aae45053021b/B in TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:14,477 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/668e0480eff441cb8abc21b4032cba27, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/0db34574bfab49e3978f2f7d0120cfcc, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/73817ec81d5f4eb19f2392140fb9f60b] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp, totalSize=35.3 K 2024-11-25T12:59:14,477 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 668e0480eff441cb8abc21b4032cba27, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732539551146 2024-11-25T12:59:14,478 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 0db34574bfab49e3978f2f7d0120cfcc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1732539551359 2024-11-25T12:59:14,478 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-25T12:59:14,478 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 73817ec81d5f4eb19f2392140fb9f60b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732539552578 2024-11-25T12:59:14,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:14,478 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2902): Flushing de45cbd8ecc09a85a2c9aae45053021b 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-25T12:59:14,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742172_1348 (size=31161) 2024-11-25T12:59:14,501 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): de45cbd8ecc09a85a2c9aae45053021b#B#compaction#293 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:59:14,502 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/B/a72d2d1a3be24c6c856d67f1fcab10c0 is 50, key is test_row_0/B:col10/1732539553287/Put/seqid=0 2024-11-25T12:59:14,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112535083c98c95b46d3909834fb6311fd6b_de45cbd8ecc09a85a2c9aae45053021b is 50, key is test_row_0/A:col10/1732539553416/Put/seqid=0 2024-11-25T12:59:14,535 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/fa0837cd25d244b89fdbe325a806773e as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/fa0837cd25d244b89fdbe325a806773e 2024-11-25T12:59:14,547 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de45cbd8ecc09a85a2c9aae45053021b/A of de45cbd8ecc09a85a2c9aae45053021b into fa0837cd25d244b89fdbe325a806773e(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:59:14,547 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de45cbd8ecc09a85a2c9aae45053021b: 2024-11-25T12:59:14,547 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., storeName=de45cbd8ecc09a85a2c9aae45053021b/A, priority=13, startTime=1732539554356; duration=0sec 2024-11-25T12:59:14,547 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:59:14,548 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de45cbd8ecc09a85a2c9aae45053021b:A 2024-11-25T12:59:14,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742174_1350 (size=12154) 2024-11-25T12:59:14,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:14,571 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112535083c98c95b46d3909834fb6311fd6b_de45cbd8ecc09a85a2c9aae45053021b to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112535083c98c95b46d3909834fb6311fd6b_de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:14,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/8c50ed3a57b34053874f94fed69e0b4b, store: [table=TestAcidGuarantees family=A region=de45cbd8ecc09a85a2c9aae45053021b] 2024-11-25T12:59:14,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/8c50ed3a57b34053874f94fed69e0b4b is 175, key is test_row_0/A:col10/1732539553416/Put/seqid=0 2024-11-25T12:59:14,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742173_1349 (size=12207) 2024-11-25T12:59:14,600 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. as already flushing 2024-11-25T12:59:14,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:14,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742175_1351 (size=30955) 2024-11-25T12:59:14,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:14,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46484 deadline: 1732539614649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:14,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:14,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:14,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:14,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:14,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46542 deadline: 1732539614661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:14,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:14,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:14,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:14,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:14,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46538 deadline: 1732539614661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:14,682 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:14,682 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:14,682 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:14,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:14,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46484 deadline: 1732539614772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:14,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:14,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:14,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:14,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:14,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46542 deadline: 1732539614790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:14,805 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:14,805 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:14,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:14,805 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:14,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46538 deadline: 1732539614790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:14,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:14,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:14,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:14,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:14,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46522 deadline: 1732539614804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:14,814 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 1ff8009e463e,35905,1732539449246 for 'test_row_1' in de45cbd8ecc09a85a2c9aae45053021b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2230 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T12:59:14,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:14,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:14,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:14,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:14,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46474 deadline: 1732539614804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:14,815 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 1ff8009e463e,35905,1732539449246 for 'test_row_2' in de45cbd8ecc09a85a2c9aae45053021b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2224 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T12:59:14,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:14,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:14,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:15,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:15,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46484 deadline: 1732539614992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:15,007 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/B/a72d2d1a3be24c6c856d67f1fcab10c0 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/a72d2d1a3be24c6c856d67f1fcab10c0 2024-11-25T12:59:15,010 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:15,010 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:15,010 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:15,012 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de45cbd8ecc09a85a2c9aae45053021b/B of de45cbd8ecc09a85a2c9aae45053021b into a72d2d1a3be24c6c856d67f1fcab10c0(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:59:15,012 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de45cbd8ecc09a85a2c9aae45053021b: 2024-11-25T12:59:15,012 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., storeName=de45cbd8ecc09a85a2c9aae45053021b/B, priority=13, startTime=1732539554356; duration=0sec 2024-11-25T12:59:15,013 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:59:15,013 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de45cbd8ecc09a85a2c9aae45053021b:B 2024-11-25T12:59:15,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:15,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46542 deadline: 1732539615010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:15,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:15,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:15,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:15,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:15,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46538 deadline: 1732539615010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:15,033 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:15,033 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:15,033 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:15,036 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=120, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/8c50ed3a57b34053874f94fed69e0b4b 2024-11-25T12:59:15,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/B/4fe14c9677664c5bafb401850e2116e4 is 50, key is test_row_0/B:col10/1732539553416/Put/seqid=0 2024-11-25T12:59:15,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742176_1352 (size=12001) 2024-11-25T12:59:15,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-25T12:59:15,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:15,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46484 deadline: 1732539615320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:15,326 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:15,326 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:15,326 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:15,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:15,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46542 deadline: 1732539615345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:15,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:15,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46538 deadline: 1732539615345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:15,348 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:15,348 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:15,348 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:15,349 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:15,349 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:15,350 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:15,480 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/B/4fe14c9677664c5bafb401850e2116e4 2024-11-25T12:59:15,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/C/da9cb6997f47465088b16f58ad38078a is 50, key is test_row_0/C:col10/1732539553416/Put/seqid=0 2024-11-25T12:59:15,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742177_1353 (size=12001) 2024-11-25T12:59:15,557 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/C/da9cb6997f47465088b16f58ad38078a 2024-11-25T12:59:15,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/8c50ed3a57b34053874f94fed69e0b4b as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/8c50ed3a57b34053874f94fed69e0b4b 2024-11-25T12:59:15,572 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/8c50ed3a57b34053874f94fed69e0b4b, entries=150, sequenceid=120, filesize=30.2 K 2024-11-25T12:59:15,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/B/4fe14c9677664c5bafb401850e2116e4 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/4fe14c9677664c5bafb401850e2116e4 2024-11-25T12:59:15,581 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/4fe14c9677664c5bafb401850e2116e4, entries=150, sequenceid=120, filesize=11.7 K 2024-11-25T12:59:15,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/C/da9cb6997f47465088b16f58ad38078a as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/da9cb6997f47465088b16f58ad38078a 2024-11-25T12:59:15,602 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/da9cb6997f47465088b16f58ad38078a, entries=150, sequenceid=120, filesize=11.7 K 2024-11-25T12:59:15,608 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for de45cbd8ecc09a85a2c9aae45053021b in 1129ms, sequenceid=120, compaction requested=false 2024-11-25T12:59:15,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2603): Flush status journal for de45cbd8ecc09a85a2c9aae45053021b: 2024-11-25T12:59:15,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:15,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=92 2024-11-25T12:59:15,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4169): Remote procedure done, pid=92 2024-11-25T12:59:15,611 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-25T12:59:15,611 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6940 sec 2024-11-25T12:59:15,613 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees in 2.6990 sec 2024-11-25T12:59:15,847 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:59:15,847 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing de45cbd8ecc09a85a2c9aae45053021b 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-25T12:59:15,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:15,879 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125acb8672d443246ac8c5d9ba389340022_de45cbd8ecc09a85a2c9aae45053021b is 50, key is test_row_0/A:col10/1732539555845/Put/seqid=0 2024-11-25T12:59:15,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742178_1354 (size=14794) 2024-11-25T12:59:15,923 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:15,926 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125acb8672d443246ac8c5d9ba389340022_de45cbd8ecc09a85a2c9aae45053021b to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125acb8672d443246ac8c5d9ba389340022_de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:15,927 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/b4f17e1a272f490a95e447470d29279f, store: [table=TestAcidGuarantees family=A region=de45cbd8ecc09a85a2c9aae45053021b] 2024-11-25T12:59:15,928 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/b4f17e1a272f490a95e447470d29279f is 175, key is test_row_0/A:col10/1732539555845/Put/seqid=0 2024-11-25T12:59:15,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742179_1355 (size=39749) 2024-11-25T12:59:15,969 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=136, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/b4f17e1a272f490a95e447470d29279f 2024-11-25T12:59:15,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:15,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46484 deadline: 1732539615976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:15,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:15,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:15,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:15,994 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/B/972f8bbfb6154d5dabb945182a608a2c is 50, key is test_row_0/B:col10/1732539555845/Put/seqid=0 2024-11-25T12:59:16,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:16,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46538 deadline: 1732539615988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:16,001 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:16,002 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:16,002 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:16,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:16,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46542 deadline: 1732539615988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:16,003 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:16,003 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:16,003 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:16,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742180_1356 (size=12151) 2024-11-25T12:59:16,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:16,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46484 deadline: 1732539616103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:16,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:16,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:16,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:16,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:16,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46542 deadline: 1732539616110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:16,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:16,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:16,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:16,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:16,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46538 deadline: 1732539616116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:16,132 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:16,132 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:16,132 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:16,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:16,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46484 deadline: 1732539616324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:16,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:16,325 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:16,325 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:16,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46542 deadline: 1732539616324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:16,325 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:16,326 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:16,326 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:16,326 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:16,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:16,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46538 deadline: 1732539616343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:16,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:16,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:16,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:16,436 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/B/972f8bbfb6154d5dabb945182a608a2c 2024-11-25T12:59:16,460 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/C/20a29e5a1b954d66bfb20df6cc9b479b is 50, key is test_row_0/C:col10/1732539555845/Put/seqid=0 2024-11-25T12:59:16,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742181_1357 (size=12151) 2024-11-25T12:59:16,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:16,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46484 deadline: 1732539616630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:16,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:16,634 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:16,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46542 deadline: 1732539616630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:16,634 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:16,634 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:16,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:16,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:16,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:16,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:16,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46538 deadline: 1732539616670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:16,680 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:16,680 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:16,680 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:16,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:16,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46474 deadline: 1732539616829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:16,839 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 1ff8009e463e,35905,1732539449246 for 'test_row_2' in de45cbd8ecc09a85a2c9aae45053021b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4248 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T12:59:16,839 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:16,839 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:16,839 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:16,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:16,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46522 deadline: 1732539616840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:16,851 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 1ff8009e463e,35905,1732539449246 for 'test_row_1' in de45cbd8ecc09a85a2c9aae45053021b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4267 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T12:59:16,851 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:16,851 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:16,851 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:16,911 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/C/20a29e5a1b954d66bfb20df6cc9b479b 2024-11-25T12:59:16,917 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/b4f17e1a272f490a95e447470d29279f as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/b4f17e1a272f490a95e447470d29279f 2024-11-25T12:59:16,920 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/b4f17e1a272f490a95e447470d29279f, entries=200, sequenceid=136, filesize=38.8 K 2024-11-25T12:59:16,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/B/972f8bbfb6154d5dabb945182a608a2c as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/972f8bbfb6154d5dabb945182a608a2c 2024-11-25T12:59:16,927 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/972f8bbfb6154d5dabb945182a608a2c, entries=150, sequenceid=136, filesize=11.9 K 2024-11-25T12:59:16,928 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/C/20a29e5a1b954d66bfb20df6cc9b479b as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/20a29e5a1b954d66bfb20df6cc9b479b 2024-11-25T12:59:16,932 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/20a29e5a1b954d66bfb20df6cc9b479b, entries=150, sequenceid=136, filesize=11.9 K 2024-11-25T12:59:16,933 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for de45cbd8ecc09a85a2c9aae45053021b in 1086ms, sequenceid=136, compaction requested=true 2024-11-25T12:59:16,933 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for de45cbd8ecc09a85a2c9aae45053021b: 2024-11-25T12:59:16,933 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:59:16,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de45cbd8ecc09a85a2c9aae45053021b:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:59:16,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:59:16,933 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:59:16,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de45cbd8ecc09a85a2c9aae45053021b:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:59:16,934 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:59:16,934 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1541): de45cbd8ecc09a85a2c9aae45053021b/A is initiating minor compaction (all files) 2024-11-25T12:59:16,934 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de45cbd8ecc09a85a2c9aae45053021b/A in TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:16,934 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/fa0837cd25d244b89fdbe325a806773e, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/8c50ed3a57b34053874f94fed69e0b4b, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/b4f17e1a272f490a95e447470d29279f] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp, totalSize=99.5 K 2024-11-25T12:59:16,934 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:16,934 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. files: [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/fa0837cd25d244b89fdbe325a806773e, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/8c50ed3a57b34053874f94fed69e0b4b, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/b4f17e1a272f490a95e447470d29279f] 2024-11-25T12:59:16,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:59:16,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de45cbd8ecc09a85a2c9aae45053021b:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:59:16,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:59:16,935 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa0837cd25d244b89fdbe325a806773e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732539552578 2024-11-25T12:59:16,935 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:59:16,935 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c50ed3a57b34053874f94fed69e0b4b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732539553406 2024-11-25T12:59:16,935 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): de45cbd8ecc09a85a2c9aae45053021b/B is initiating minor compaction (all files) 2024-11-25T12:59:16,935 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de45cbd8ecc09a85a2c9aae45053021b/B in TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:16,935 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/a72d2d1a3be24c6c856d67f1fcab10c0, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/4fe14c9677664c5bafb401850e2116e4, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/972f8bbfb6154d5dabb945182a608a2c] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp, totalSize=35.5 K 2024-11-25T12:59:16,935 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4f17e1a272f490a95e447470d29279f, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732539554628 2024-11-25T12:59:16,935 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting a72d2d1a3be24c6c856d67f1fcab10c0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732539552578 2024-11-25T12:59:16,937 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 4fe14c9677664c5bafb401850e2116e4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732539553406 2024-11-25T12:59:16,939 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 972f8bbfb6154d5dabb945182a608a2c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732539554628 2024-11-25T12:59:16,949 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): de45cbd8ecc09a85a2c9aae45053021b#B#compaction#300 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:59:16,949 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/B/483b294026f34b5f8428b367f2ce7b8c is 50, key is test_row_0/B:col10/1732539555845/Put/seqid=0 2024-11-25T12:59:16,952 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=de45cbd8ecc09a85a2c9aae45053021b] 2024-11-25T12:59:16,972 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241125c1882f2cab7d4f2c81a78489ce4df18e_de45cbd8ecc09a85a2c9aae45053021b store=[table=TestAcidGuarantees family=A region=de45cbd8ecc09a85a2c9aae45053021b] 2024-11-25T12:59:16,974 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241125c1882f2cab7d4f2c81a78489ce4df18e_de45cbd8ecc09a85a2c9aae45053021b, store=[table=TestAcidGuarantees family=A region=de45cbd8ecc09a85a2c9aae45053021b] 2024-11-25T12:59:16,974 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125c1882f2cab7d4f2c81a78489ce4df18e_de45cbd8ecc09a85a2c9aae45053021b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=de45cbd8ecc09a85a2c9aae45053021b] 2024-11-25T12:59:16,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742182_1358 (size=12459) 2024-11-25T12:59:17,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742183_1359 (size=4469) 2024-11-25T12:59:17,034 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): de45cbd8ecc09a85a2c9aae45053021b#A#compaction#301 average throughput is 0.30 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:59:17,035 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/a4934b14da9048d69dae045b8c44fee3 is 175, key is test_row_0/A:col10/1732539555845/Put/seqid=0 2024-11-25T12:59:17,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742184_1360 (size=31413) 2024-11-25T12:59:17,086 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/a4934b14da9048d69dae045b8c44fee3 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/a4934b14da9048d69dae045b8c44fee3 2024-11-25T12:59:17,094 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de45cbd8ecc09a85a2c9aae45053021b/A of de45cbd8ecc09a85a2c9aae45053021b into a4934b14da9048d69dae045b8c44fee3(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:59:17,094 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de45cbd8ecc09a85a2c9aae45053021b: 2024-11-25T12:59:17,094 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., storeName=de45cbd8ecc09a85a2c9aae45053021b/A, priority=13, startTime=1732539556933; duration=0sec 2024-11-25T12:59:17,094 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:59:17,094 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de45cbd8ecc09a85a2c9aae45053021b:A 2024-11-25T12:59:17,094 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:59:17,095 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:59:17,095 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1541): de45cbd8ecc09a85a2c9aae45053021b/C is initiating minor compaction (all files) 2024-11-25T12:59:17,096 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de45cbd8ecc09a85a2c9aae45053021b/C in TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:17,096 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/dc5a19e2d9fa4ced956635a857081496, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/da9cb6997f47465088b16f58ad38078a, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/20a29e5a1b954d66bfb20df6cc9b479b] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp, totalSize=35.5 K 2024-11-25T12:59:17,096 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc5a19e2d9fa4ced956635a857081496, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732539552578 2024-11-25T12:59:17,096 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting da9cb6997f47465088b16f58ad38078a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732539553406 2024-11-25T12:59:17,097 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20a29e5a1b954d66bfb20df6cc9b479b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732539554628 2024-11-25T12:59:17,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-25T12:59:17,112 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:59:17,112 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:59:17,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-11-25T12:59:17,114 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:59:17,115 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:59:17,116 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:59:17,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-25T12:59:17,125 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): de45cbd8ecc09a85a2c9aae45053021b#C#compaction#302 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:59:17,126 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/C/c722365822cc4185a1742d01698b2b1f is 50, key is test_row_0/C:col10/1732539555845/Put/seqid=0 2024-11-25T12:59:17,147 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:59:17,147 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing de45cbd8ecc09a85a2c9aae45053021b 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-25T12:59:17,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:17,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742185_1361 (size=12459) 2024-11-25T12:59:17,185 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411250e0239cb476843a597cee69ef27d2276_de45cbd8ecc09a85a2c9aae45053021b is 50, key is test_row_0/A:col10/1732539555968/Put/seqid=0 2024-11-25T12:59:17,186 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/C/c722365822cc4185a1742d01698b2b1f as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/c722365822cc4185a1742d01698b2b1f 2024-11-25T12:59:17,196 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de45cbd8ecc09a85a2c9aae45053021b/C of de45cbd8ecc09a85a2c9aae45053021b into c722365822cc4185a1742d01698b2b1f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:59:17,196 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de45cbd8ecc09a85a2c9aae45053021b: 2024-11-25T12:59:17,196 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., storeName=de45cbd8ecc09a85a2c9aae45053021b/C, priority=13, startTime=1732539556934; duration=0sec 2024-11-25T12:59:17,197 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:59:17,197 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de45cbd8ecc09a85a2c9aae45053021b:C 2024-11-25T12:59:17,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742186_1362 (size=14794) 2024-11-25T12:59:17,238 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:17,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-25T12:59:17,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:17,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46542 deadline: 1732539617216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:17,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:17,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46484 deadline: 1732539617217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:17,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:17,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46538 deadline: 1732539617224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:17,244 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:17,244 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:17,244 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:17,244 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:17,244 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:17,244 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:17,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:17,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:17,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:17,254 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411250e0239cb476843a597cee69ef27d2276_de45cbd8ecc09a85a2c9aae45053021b to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411250e0239cb476843a597cee69ef27d2276_de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:17,256 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/02639827a95e4c91a59192950db51610, store: [table=TestAcidGuarantees family=A region=de45cbd8ecc09a85a2c9aae45053021b] 2024-11-25T12:59:17,257 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/02639827a95e4c91a59192950db51610 is 175, key is test_row_0/A:col10/1732539555968/Put/seqid=0 2024-11-25T12:59:17,288 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-25T12:59:17,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:17,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. as already flushing 2024-11-25T12:59:17,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:17,288 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:17,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:17,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:17,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742187_1363 (size=39749) 2024-11-25T12:59:17,301 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=160, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/02639827a95e4c91a59192950db51610 2024-11-25T12:59:17,313 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/B/e6110a0b04d94e5eb794ff68941ecb85 is 50, key is test_row_0/B:col10/1732539555968/Put/seqid=0 2024-11-25T12:59:17,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:17,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46542 deadline: 1732539617350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:17,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:17,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:17,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:17,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:17,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46484 deadline: 1732539617350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:17,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:17,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:17,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:17,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:17,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46538 deadline: 1732539617350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:17,357 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:17,358 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:17,358 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:17,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742188_1364 (size=12151) 2024-11-25T12:59:17,403 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/B/483b294026f34b5f8428b367f2ce7b8c as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/483b294026f34b5f8428b367f2ce7b8c 2024-11-25T12:59:17,413 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de45cbd8ecc09a85a2c9aae45053021b/B of de45cbd8ecc09a85a2c9aae45053021b into 483b294026f34b5f8428b367f2ce7b8c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:59:17,413 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de45cbd8ecc09a85a2c9aae45053021b: 2024-11-25T12:59:17,413 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., storeName=de45cbd8ecc09a85a2c9aae45053021b/B, priority=13, startTime=1732539556933; duration=0sec 2024-11-25T12:59:17,413 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:59:17,413 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de45cbd8ecc09a85a2c9aae45053021b:B 2024-11-25T12:59:17,442 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-25T12:59:17,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:17,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. as already flushing 2024-11-25T12:59:17,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:17,443 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:17,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:17,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:17,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-25T12:59:17,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:17,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46538 deadline: 1732539617560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:17,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:17,568 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:17,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46542 deadline: 1732539617560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:17,568 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:17,568 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:17,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:17,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:17,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:17,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:17,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46484 deadline: 1732539617569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:17,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:17,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:17,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:17,603 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-25T12:59:17,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:17,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. as already flushing 2024-11-25T12:59:17,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:17,604 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:17,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:17,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:17,757 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-25T12:59:17,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:17,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. as already flushing 2024-11-25T12:59:17,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:17,757 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:17,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:17,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:17,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-25T12:59:17,784 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/B/e6110a0b04d94e5eb794ff68941ecb85 2024-11-25T12:59:17,799 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/C/6f9c9ab5a6c849efaf93edfb28ccd599 is 50, key is test_row_0/C:col10/1732539555968/Put/seqid=0 2024-11-25T12:59:17,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742189_1365 (size=12151) 2024-11-25T12:59:17,849 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/C/6f9c9ab5a6c849efaf93edfb28ccd599 2024-11-25T12:59:17,858 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/02639827a95e4c91a59192950db51610 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/02639827a95e4c91a59192950db51610 2024-11-25T12:59:17,862 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/02639827a95e4c91a59192950db51610, entries=200, sequenceid=160, filesize=38.8 K 2024-11-25T12:59:17,863 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/B/e6110a0b04d94e5eb794ff68941ecb85 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/e6110a0b04d94e5eb794ff68941ecb85 2024-11-25T12:59:17,866 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/e6110a0b04d94e5eb794ff68941ecb85, entries=150, sequenceid=160, filesize=11.9 K 2024-11-25T12:59:17,867 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/C/6f9c9ab5a6c849efaf93edfb28ccd599 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/6f9c9ab5a6c849efaf93edfb28ccd599 2024-11-25T12:59:17,870 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/6f9c9ab5a6c849efaf93edfb28ccd599, entries=150, sequenceid=160, filesize=11.9 K 2024-11-25T12:59:17,877 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for de45cbd8ecc09a85a2c9aae45053021b in 730ms, sequenceid=160, compaction requested=false 2024-11-25T12:59:17,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for de45cbd8ecc09a85a2c9aae45053021b: 2024-11-25T12:59:17,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:17,884 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:59:17,884 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing de45cbd8ecc09a85a2c9aae45053021b 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-25T12:59:17,900 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125c2e7f95f407e4bdbb64bb9b096b838b3_de45cbd8ecc09a85a2c9aae45053021b is 50, key is test_row_0/A:col10/1732539557209/Put/seqid=0 2024-11-25T12:59:17,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742190_1366 (size=14794) 2024-11-25T12:59:17,904 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:17,909 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125c2e7f95f407e4bdbb64bb9b096b838b3_de45cbd8ecc09a85a2c9aae45053021b to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125c2e7f95f407e4bdbb64bb9b096b838b3_de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:17,910 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/e76487964ff84cc88388f533f8090a7c, store: [table=TestAcidGuarantees family=A region=de45cbd8ecc09a85a2c9aae45053021b] 2024-11-25T12:59:17,910 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/e76487964ff84cc88388f533f8090a7c is 175, key is test_row_0/A:col10/1732539557209/Put/seqid=0 2024-11-25T12:59:17,912 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-25T12:59:17,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:17,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. as already flushing 2024-11-25T12:59:17,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:17,912 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:17,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:17,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:17,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742191_1367 (size=39749) 2024-11-25T12:59:17,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:17,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46538 deadline: 1732539617961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:17,974 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:17,974 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:17,974 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:17,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:17,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46542 deadline: 1732539617959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:17,975 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:17,975 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:17,975 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:17,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:17,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46484 deadline: 1732539617966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:17,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:17,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:17,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:18,065 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-25T12:59:18,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:18,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. as already flushing 2024-11-25T12:59:18,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:18,065 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:18,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:18,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:18,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:18,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46538 deadline: 1732539618080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:18,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:18,087 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:18,087 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:18,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46542 deadline: 1732539618080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:18,087 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:18,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:18,087 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:18,087 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:18,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46484 deadline: 1732539618080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:18,087 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:18,087 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:18,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:18,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:18,220 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-25T12:59:18,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:18,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. as already flushing 2024-11-25T12:59:18,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:18,220 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:18,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:18,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:18,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-25T12:59:18,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:18,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46542 deadline: 1732539618290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:18,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:18,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:18,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46484 deadline: 1732539618290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:18,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:18,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:18,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:18,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:18,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:18,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:18,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46538 deadline: 1732539618299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:18,302 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:18,302 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:18,302 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:18,336 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=176, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/e76487964ff84cc88388f533f8090a7c 2024-11-25T12:59:18,353 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/B/251162e0708a4c62a83edf9ff307425d is 50, key is test_row_0/B:col10/1732539557209/Put/seqid=0 2024-11-25T12:59:18,385 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-25T12:59:18,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:18,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. as already flushing 2024-11-25T12:59:18,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:18,386 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:18,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:18,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:18,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742192_1368 (size=12151) 2024-11-25T12:59:18,400 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/B/251162e0708a4c62a83edf9ff307425d 2024-11-25T12:59:18,413 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/C/0128002f234b476a96ab44c3673d9c40 is 50, key is test_row_0/C:col10/1732539557209/Put/seqid=0 2024-11-25T12:59:18,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742193_1369 (size=12151) 2024-11-25T12:59:18,457 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/C/0128002f234b476a96ab44c3673d9c40 2024-11-25T12:59:18,466 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/e76487964ff84cc88388f533f8090a7c as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/e76487964ff84cc88388f533f8090a7c 2024-11-25T12:59:18,476 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/e76487964ff84cc88388f533f8090a7c, entries=200, sequenceid=176, filesize=38.8 K 2024-11-25T12:59:18,479 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/B/251162e0708a4c62a83edf9ff307425d as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/251162e0708a4c62a83edf9ff307425d 2024-11-25T12:59:18,494 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/251162e0708a4c62a83edf9ff307425d, entries=150, sequenceid=176, filesize=11.9 K 2024-11-25T12:59:18,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/C/0128002f234b476a96ab44c3673d9c40 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/0128002f234b476a96ab44c3673d9c40 2024-11-25T12:59:18,499 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/0128002f234b476a96ab44c3673d9c40, entries=150, sequenceid=176, filesize=11.9 K 2024-11-25T12:59:18,500 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for de45cbd8ecc09a85a2c9aae45053021b in 616ms, sequenceid=176, compaction requested=true 2024-11-25T12:59:18,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for de45cbd8ecc09a85a2c9aae45053021b: 2024-11-25T12:59:18,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de45cbd8ecc09a85a2c9aae45053021b:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:59:18,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:59:18,500 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:59:18,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de45cbd8ecc09a85a2c9aae45053021b:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:59:18,501 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:59:18,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:59:18,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de45cbd8ecc09a85a2c9aae45053021b:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:59:18,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:59:18,503 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110911 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:59:18,503 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1541): de45cbd8ecc09a85a2c9aae45053021b/A is initiating minor compaction (all files) 2024-11-25T12:59:18,503 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de45cbd8ecc09a85a2c9aae45053021b/A in TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:18,503 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/a4934b14da9048d69dae045b8c44fee3, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/02639827a95e4c91a59192950db51610, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/e76487964ff84cc88388f533f8090a7c] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp, totalSize=108.3 K 2024-11-25T12:59:18,503 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:18,503 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. files: [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/a4934b14da9048d69dae045b8c44fee3, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/02639827a95e4c91a59192950db51610, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/e76487964ff84cc88388f533f8090a7c] 2024-11-25T12:59:18,503 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:59:18,503 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): de45cbd8ecc09a85a2c9aae45053021b/B is initiating minor compaction (all files) 2024-11-25T12:59:18,503 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de45cbd8ecc09a85a2c9aae45053021b/B in TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:18,503 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/483b294026f34b5f8428b367f2ce7b8c, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/e6110a0b04d94e5eb794ff68941ecb85, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/251162e0708a4c62a83edf9ff307425d] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp, totalSize=35.9 K 2024-11-25T12:59:18,503 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4934b14da9048d69dae045b8c44fee3, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732539554628 2024-11-25T12:59:18,504 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 483b294026f34b5f8428b367f2ce7b8c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732539554628 2024-11-25T12:59:18,504 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 02639827a95e4c91a59192950db51610, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1732539555963 2024-11-25T12:59:18,504 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting e6110a0b04d94e5eb794ff68941ecb85, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1732539555968 2024-11-25T12:59:18,504 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting e76487964ff84cc88388f533f8090a7c, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732539557199 2024-11-25T12:59:18,504 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 251162e0708a4c62a83edf9ff307425d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732539557209 2024-11-25T12:59:18,526 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): de45cbd8ecc09a85a2c9aae45053021b#B#compaction#309 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:59:18,526 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/B/932c28f3872d4636a97c50e4b091c3cc is 50, key is test_row_0/B:col10/1732539557209/Put/seqid=0 2024-11-25T12:59:18,536 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=de45cbd8ecc09a85a2c9aae45053021b] 2024-11-25T12:59:18,554 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-25T12:59:18,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:18,555 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing de45cbd8ecc09a85a2c9aae45053021b 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-25T12:59:18,576 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241125e45779b02486496ea95a0a4087d2ab04_de45cbd8ecc09a85a2c9aae45053021b store=[table=TestAcidGuarantees family=A region=de45cbd8ecc09a85a2c9aae45053021b] 2024-11-25T12:59:18,578 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241125e45779b02486496ea95a0a4087d2ab04_de45cbd8ecc09a85a2c9aae45053021b, store=[table=TestAcidGuarantees family=A region=de45cbd8ecc09a85a2c9aae45053021b] 2024-11-25T12:59:18,578 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125e45779b02486496ea95a0a4087d2ab04_de45cbd8ecc09a85a2c9aae45053021b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=de45cbd8ecc09a85a2c9aae45053021b] 2024-11-25T12:59:18,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742194_1370 (size=12561) 2024-11-25T12:59:18,619 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. as already flushing 2024-11-25T12:59:18,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:18,626 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/B/932c28f3872d4636a97c50e4b091c3cc as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/932c28f3872d4636a97c50e4b091c3cc 2024-11-25T12:59:18,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125dd54718c1cf84a418eb3c1b85d53ec53_de45cbd8ecc09a85a2c9aae45053021b is 50, key is test_row_0/A:col10/1732539557913/Put/seqid=0 2024-11-25T12:59:18,633 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de45cbd8ecc09a85a2c9aae45053021b/B of de45cbd8ecc09a85a2c9aae45053021b into 932c28f3872d4636a97c50e4b091c3cc(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:59:18,633 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de45cbd8ecc09a85a2c9aae45053021b: 2024-11-25T12:59:18,633 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., storeName=de45cbd8ecc09a85a2c9aae45053021b/B, priority=13, startTime=1732539558501; duration=0sec 2024-11-25T12:59:18,633 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:59:18,633 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de45cbd8ecc09a85a2c9aae45053021b:B 2024-11-25T12:59:18,633 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:59:18,634 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:59:18,634 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): de45cbd8ecc09a85a2c9aae45053021b/C is initiating minor compaction (all files) 2024-11-25T12:59:18,634 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de45cbd8ecc09a85a2c9aae45053021b/C in TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:18,634 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/c722365822cc4185a1742d01698b2b1f, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/6f9c9ab5a6c849efaf93edfb28ccd599, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/0128002f234b476a96ab44c3673d9c40] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp, totalSize=35.9 K 2024-11-25T12:59:18,635 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting c722365822cc4185a1742d01698b2b1f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732539554628 2024-11-25T12:59:18,635 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f9c9ab5a6c849efaf93edfb28ccd599, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1732539555968 2024-11-25T12:59:18,635 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 0128002f234b476a96ab44c3673d9c40, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732539557209 2024-11-25T12:59:18,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742196_1372 (size=12304) 2024-11-25T12:59:18,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742195_1371 (size=4469) 2024-11-25T12:59:18,649 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): de45cbd8ecc09a85a2c9aae45053021b#A#compaction#310 average throughput is 0.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:59:18,650 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/5b28e5a0b9234bd9ba0db1b56e34e29a is 175, key is test_row_0/A:col10/1732539557209/Put/seqid=0 2024-11-25T12:59:18,666 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): de45cbd8ecc09a85a2c9aae45053021b#C#compaction#312 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:59:18,666 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/C/d47d83e411454e1dafb3cba5dd35d173 is 50, key is test_row_0/C:col10/1732539557209/Put/seqid=0 2024-11-25T12:59:18,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:18,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46542 deadline: 1732539618657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:18,691 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:18,691 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:18,691 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:18,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:18,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46538 deadline: 1732539618668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:18,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:18,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46484 deadline: 1732539618676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:18,694 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:18,694 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:18,694 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:18,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:18,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:18,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:18,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742197_1373 (size=31515) 2024-11-25T12:59:18,719 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/5b28e5a0b9234bd9ba0db1b56e34e29a as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/5b28e5a0b9234bd9ba0db1b56e34e29a 2024-11-25T12:59:18,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742198_1374 (size=12561) 2024-11-25T12:59:18,731 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de45cbd8ecc09a85a2c9aae45053021b/A of de45cbd8ecc09a85a2c9aae45053021b into 5b28e5a0b9234bd9ba0db1b56e34e29a(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:59:18,732 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de45cbd8ecc09a85a2c9aae45053021b: 2024-11-25T12:59:18,732 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., storeName=de45cbd8ecc09a85a2c9aae45053021b/A, priority=13, startTime=1732539558500; duration=0sec 2024-11-25T12:59:18,732 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:59:18,732 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de45cbd8ecc09a85a2c9aae45053021b:A 2024-11-25T12:59:18,775 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/C/d47d83e411454e1dafb3cba5dd35d173 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/d47d83e411454e1dafb3cba5dd35d173 2024-11-25T12:59:18,810 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de45cbd8ecc09a85a2c9aae45053021b/C of de45cbd8ecc09a85a2c9aae45053021b into d47d83e411454e1dafb3cba5dd35d173(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:59:18,810 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de45cbd8ecc09a85a2c9aae45053021b: 2024-11-25T12:59:18,810 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., storeName=de45cbd8ecc09a85a2c9aae45053021b/C, priority=13, startTime=1732539558501; duration=0sec 2024-11-25T12:59:18,810 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:59:18,810 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de45cbd8ecc09a85a2c9aae45053021b:C 2024-11-25T12:59:18,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:18,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46542 deadline: 1732539618800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:18,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:18,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:18,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46538 deadline: 1732539618800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:18,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:18,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:18,812 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:18,812 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:18,812 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:18,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:18,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46484 deadline: 1732539618800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:18,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:18,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:18,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:19,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:19,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46542 deadline: 1732539619020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:19,037 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:19,037 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:19,038 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:19,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:19,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46538 deadline: 1732539619020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:19,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:19,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:19,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:19,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:19,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46484 deadline: 1732539619021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:19,038 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:19,038 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:19,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:19,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:19,049 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125dd54718c1cf84a418eb3c1b85d53ec53_de45cbd8ecc09a85a2c9aae45053021b to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125dd54718c1cf84a418eb3c1b85d53ec53_de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:19,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/17d9d87b463649bea96531fcce85fab0, store: [table=TestAcidGuarantees family=A region=de45cbd8ecc09a85a2c9aae45053021b] 2024-11-25T12:59:19,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/17d9d87b463649bea96531fcce85fab0 is 175, key is test_row_0/A:col10/1732539557913/Put/seqid=0 2024-11-25T12:59:19,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742199_1375 (size=31105) 2024-11-25T12:59:19,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-25T12:59:19,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:19,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46542 deadline: 1732539619346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:19,349 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:19,349 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:19,349 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:19,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:19,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46538 deadline: 1732539619354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:19,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:19,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:19,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:19,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:19,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46484 deadline: 1732539619354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:19,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:19,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:19,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:19,508 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=198, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/17d9d87b463649bea96531fcce85fab0 2024-11-25T12:59:19,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/B/993d67a154834ce08fe73b31657910ca is 50, key is test_row_0/B:col10/1732539557913/Put/seqid=0 2024-11-25T12:59:19,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742200_1376 (size=12151) 2024-11-25T12:59:19,603 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/B/993d67a154834ce08fe73b31657910ca 2024-11-25T12:59:19,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/C/9968c6ebf6f3465fbd2aa1386f0da5b3 is 50, key is test_row_0/C:col10/1732539557913/Put/seqid=0 2024-11-25T12:59:19,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742201_1377 (size=12151) 2024-11-25T12:59:19,645 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/C/9968c6ebf6f3465fbd2aa1386f0da5b3 2024-11-25T12:59:19,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/17d9d87b463649bea96531fcce85fab0 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/17d9d87b463649bea96531fcce85fab0 2024-11-25T12:59:19,661 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/17d9d87b463649bea96531fcce85fab0, entries=150, sequenceid=198, filesize=30.4 K 2024-11-25T12:59:19,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/B/993d67a154834ce08fe73b31657910ca as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/993d67a154834ce08fe73b31657910ca 2024-11-25T12:59:19,680 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/993d67a154834ce08fe73b31657910ca, entries=150, sequenceid=198, filesize=11.9 K 2024-11-25T12:59:19,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/C/9968c6ebf6f3465fbd2aa1386f0da5b3 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/9968c6ebf6f3465fbd2aa1386f0da5b3 2024-11-25T12:59:19,688 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/9968c6ebf6f3465fbd2aa1386f0da5b3, entries=150, sequenceid=198, filesize=11.9 K 2024-11-25T12:59:19,689 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for de45cbd8ecc09a85a2c9aae45053021b in 1134ms, sequenceid=198, compaction requested=false 2024-11-25T12:59:19,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for de45cbd8ecc09a85a2c9aae45053021b: 2024-11-25T12:59:19,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:19,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-11-25T12:59:19,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-11-25T12:59:19,704 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-25T12:59:19,704 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5780 sec 2024-11-25T12:59:19,707 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 2.5940 sec 2024-11-25T12:59:19,873 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:59:19,873 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing de45cbd8ecc09a85a2c9aae45053021b 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-25T12:59:19,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:19,903 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125da1bf51813294678a46282b3b181a118_de45cbd8ecc09a85a2c9aae45053021b is 50, key is test_row_0/A:col10/1732539558653/Put/seqid=0 2024-11-25T12:59:19,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742202_1378 (size=14794) 2024-11-25T12:59:19,912 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:19,920 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125da1bf51813294678a46282b3b181a118_de45cbd8ecc09a85a2c9aae45053021b to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125da1bf51813294678a46282b3b181a118_de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:19,922 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/fe9f657cb5f147aa80f33c2e4e2ec001, store: [table=TestAcidGuarantees family=A region=de45cbd8ecc09a85a2c9aae45053021b] 2024-11-25T12:59:19,922 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/fe9f657cb5f147aa80f33c2e4e2ec001 is 175, key is test_row_0/A:col10/1732539558653/Put/seqid=0 2024-11-25T12:59:19,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742203_1379 (size=39749) 2024-11-25T12:59:20,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:20,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46538 deadline: 1732539620033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:20,049 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:20,049 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:20,049 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:20,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:20,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46542 deadline: 1732539620035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:20,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:20,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:20,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:20,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:20,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46484 deadline: 1732539620044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:20,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:20,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:20,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:20,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:20,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46538 deadline: 1732539620160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:20,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:20,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:20,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:20,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:20,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46542 deadline: 1732539620160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:20,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:20,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46484 deadline: 1732539620161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:20,167 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:20,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:20,167 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:20,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:20,168 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:20,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:20,363 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=216, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/fe9f657cb5f147aa80f33c2e4e2ec001 2024-11-25T12:59:20,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:20,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46538 deadline: 1732539620370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:20,374 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:20,374 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:20,374 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:20,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:20,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46542 deadline: 1732539620370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:20,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:20,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:20,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:20,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:20,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46484 deadline: 1732539620370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:20,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:20,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:20,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:20,381 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/B/eac9cd5f0256494ea803f9fdd86f3103 is 50, key is test_row_0/B:col10/1732539558653/Put/seqid=0 2024-11-25T12:59:20,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742204_1380 (size=12151) 2024-11-25T12:59:20,432 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/B/eac9cd5f0256494ea803f9fdd86f3103 2024-11-25T12:59:20,477 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/C/70979e5b912d4edca01e353d15bd864e is 50, key is test_row_0/C:col10/1732539558653/Put/seqid=0 2024-11-25T12:59:20,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742205_1381 (size=12151) 2024-11-25T12:59:20,551 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/C/70979e5b912d4edca01e353d15bd864e 2024-11-25T12:59:20,588 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/fe9f657cb5f147aa80f33c2e4e2ec001 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/fe9f657cb5f147aa80f33c2e4e2ec001 2024-11-25T12:59:20,618 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/fe9f657cb5f147aa80f33c2e4e2ec001, entries=200, sequenceid=216, filesize=38.8 K 2024-11-25T12:59:20,627 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/B/eac9cd5f0256494ea803f9fdd86f3103 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/eac9cd5f0256494ea803f9fdd86f3103 2024-11-25T12:59:20,658 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/eac9cd5f0256494ea803f9fdd86f3103, entries=150, sequenceid=216, filesize=11.9 K 2024-11-25T12:59:20,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/C/70979e5b912d4edca01e353d15bd864e as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/70979e5b912d4edca01e353d15bd864e 2024-11-25T12:59:20,669 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/70979e5b912d4edca01e353d15bd864e, entries=150, sequenceid=216, filesize=11.9 K 2024-11-25T12:59:20,671 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for de45cbd8ecc09a85a2c9aae45053021b in 797ms, sequenceid=216, compaction requested=true 2024-11-25T12:59:20,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for de45cbd8ecc09a85a2c9aae45053021b: 2024-11-25T12:59:20,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de45cbd8ecc09a85a2c9aae45053021b:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:59:20,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:59:20,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de45cbd8ecc09a85a2c9aae45053021b:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:59:20,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-25T12:59:20,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de45cbd8ecc09a85a2c9aae45053021b:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:59:20,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-25T12:59:20,671 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:59:20,671 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:59:20,676 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102369 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:59:20,677 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1541): de45cbd8ecc09a85a2c9aae45053021b/A is initiating minor compaction (all files) 2024-11-25T12:59:20,677 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de45cbd8ecc09a85a2c9aae45053021b/A in TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:20,677 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/5b28e5a0b9234bd9ba0db1b56e34e29a, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/17d9d87b463649bea96531fcce85fab0, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/fe9f657cb5f147aa80f33c2e4e2ec001] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp, totalSize=100.0 K 2024-11-25T12:59:20,677 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:20,677 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. files: [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/5b28e5a0b9234bd9ba0db1b56e34e29a, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/17d9d87b463649bea96531fcce85fab0, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/fe9f657cb5f147aa80f33c2e4e2ec001] 2024-11-25T12:59:20,677 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:59:20,677 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): de45cbd8ecc09a85a2c9aae45053021b/C is initiating minor compaction (all files) 2024-11-25T12:59:20,677 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de45cbd8ecc09a85a2c9aae45053021b/C in TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:20,677 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b28e5a0b9234bd9ba0db1b56e34e29a, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732539557209 2024-11-25T12:59:20,677 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/d47d83e411454e1dafb3cba5dd35d173, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/9968c6ebf6f3465fbd2aa1386f0da5b3, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/70979e5b912d4edca01e353d15bd864e] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp, totalSize=36.0 K 2024-11-25T12:59:20,678 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting d47d83e411454e1dafb3cba5dd35d173, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732539557209 2024-11-25T12:59:20,678 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17d9d87b463649bea96531fcce85fab0, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732539557913 2024-11-25T12:59:20,678 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe9f657cb5f147aa80f33c2e4e2ec001, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732539558640 2024-11-25T12:59:20,678 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 9968c6ebf6f3465fbd2aa1386f0da5b3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732539557913 2024-11-25T12:59:20,678 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 70979e5b912d4edca01e353d15bd864e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732539558640 2024-11-25T12:59:20,697 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=de45cbd8ecc09a85a2c9aae45053021b] 2024-11-25T12:59:20,697 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): de45cbd8ecc09a85a2c9aae45053021b#C#compaction#318 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:59:20,698 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/C/1e689ab59b6940a1a3fe7a317284fa97 is 50, key is test_row_0/C:col10/1732539558653/Put/seqid=0 2024-11-25T12:59:20,700 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:59:20,700 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing de45cbd8ecc09a85a2c9aae45053021b 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-25T12:59:20,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:20,705 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241125231cffd6a74d421eb31d4e2ccbba2d06_de45cbd8ecc09a85a2c9aae45053021b store=[table=TestAcidGuarantees family=A region=de45cbd8ecc09a85a2c9aae45053021b] 2024-11-25T12:59:20,707 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241125231cffd6a74d421eb31d4e2ccbba2d06_de45cbd8ecc09a85a2c9aae45053021b, store=[table=TestAcidGuarantees family=A region=de45cbd8ecc09a85a2c9aae45053021b] 2024-11-25T12:59:20,707 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125231cffd6a74d421eb31d4e2ccbba2d06_de45cbd8ecc09a85a2c9aae45053021b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=de45cbd8ecc09a85a2c9aae45053021b] 2024-11-25T12:59:20,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742206_1382 (size=12663) 2024-11-25T12:59:20,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742207_1383 (size=4469) 2024-11-25T12:59:20,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:20,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46542 deadline: 1732539620767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:20,791 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:20,791 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:20,791 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:20,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:20,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:20,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46484 deadline: 1732539620768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:20,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46538 deadline: 1732539620773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:20,792 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:20,792 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:20,792 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:20,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:20,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:20,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:20,801 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125014dfe5576754de8aba4bcbbfbc2be95_de45cbd8ecc09a85a2c9aae45053021b is 50, key is test_row_0/A:col10/1732539560042/Put/seqid=0 2024-11-25T12:59:20,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742208_1384 (size=12304) 2024-11-25T12:59:20,837 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:20,841 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125014dfe5576754de8aba4bcbbfbc2be95_de45cbd8ecc09a85a2c9aae45053021b to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125014dfe5576754de8aba4bcbbfbc2be95_de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:20,842 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/1cab360fd6bd4cf9a9b6cdb95e1f750e, store: [table=TestAcidGuarantees family=A region=de45cbd8ecc09a85a2c9aae45053021b] 2024-11-25T12:59:20,849 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/1cab360fd6bd4cf9a9b6cdb95e1f750e is 175, key is test_row_0/A:col10/1732539560042/Put/seqid=0 2024-11-25T12:59:20,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742209_1385 (size=31105) 2024-11-25T12:59:20,877 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=239, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/1cab360fd6bd4cf9a9b6cdb95e1f750e 2024-11-25T12:59:20,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:20,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46522 deadline: 1732539620869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:20,878 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 1ff8009e463e,35905,1732539449246 for 'test_row_1' in de45cbd8ecc09a85a2c9aae45053021b of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8294 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T12:59:20,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:20,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:20,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:20,889 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/B/7dc7b6cf2fe140acb223a019be6e470e is 50, key is test_row_0/B:col10/1732539560042/Put/seqid=0 2024-11-25T12:59:20,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:20,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46474 deadline: 1732539620880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:20,890 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 1ff8009e463e,35905,1732539449246 for 'test_row_2' in de45cbd8ecc09a85a2c9aae45053021b of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8299 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T12:59:20,891 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:20,891 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:20,891 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:20,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742210_1386 (size=12151) 2024-11-25T12:59:20,903 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/B/7dc7b6cf2fe140acb223a019be6e470e 2024-11-25T12:59:20,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:20,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46538 deadline: 1732539620900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:20,908 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:20,908 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:20,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:20,908 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:20,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46484 deadline: 1732539620900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:20,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:20,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:20,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:20,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:20,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46542 deadline: 1732539620900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:20,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:20,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:20,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:20,929 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/C/143f1559c61540bf935b24eba0dce8ed is 50, key is test_row_0/C:col10/1732539560042/Put/seqid=0 2024-11-25T12:59:20,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742211_1387 (size=12151) 2024-11-25T12:59:20,956 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/C/143f1559c61540bf935b24eba0dce8ed 2024-11-25T12:59:20,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/1cab360fd6bd4cf9a9b6cdb95e1f750e as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/1cab360fd6bd4cf9a9b6cdb95e1f750e 2024-11-25T12:59:20,969 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/1cab360fd6bd4cf9a9b6cdb95e1f750e, entries=150, sequenceid=239, filesize=30.4 K 2024-11-25T12:59:20,970 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/B/7dc7b6cf2fe140acb223a019be6e470e as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/7dc7b6cf2fe140acb223a019be6e470e 2024-11-25T12:59:20,973 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/7dc7b6cf2fe140acb223a019be6e470e, entries=150, sequenceid=239, filesize=11.9 K 2024-11-25T12:59:20,974 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/C/143f1559c61540bf935b24eba0dce8ed as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/143f1559c61540bf935b24eba0dce8ed 2024-11-25T12:59:20,980 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/143f1559c61540bf935b24eba0dce8ed, entries=150, sequenceid=239, filesize=11.9 K 2024-11-25T12:59:20,981 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for de45cbd8ecc09a85a2c9aae45053021b in 281ms, sequenceid=239, compaction requested=true 2024-11-25T12:59:20,981 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for de45cbd8ecc09a85a2c9aae45053021b: 2024-11-25T12:59:20,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de45cbd8ecc09a85a2c9aae45053021b:A, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:59:20,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-25T12:59:20,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de45cbd8ecc09a85a2c9aae45053021b:B, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:59:20,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-25T12:59:20,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de45cbd8ecc09a85a2c9aae45053021b:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:59:20,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-25T12:59:21,115 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:59:21,116 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing de45cbd8ecc09a85a2c9aae45053021b 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-25T12:59:21,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:21,129 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112586974719299c48bd94d8c16fdf840002_de45cbd8ecc09a85a2c9aae45053021b is 50, key is test_row_0/A:col10/1732539560767/Put/seqid=0 2024-11-25T12:59:21,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742212_1388 (size=14794) 2024-11-25T12:59:21,171 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/C/1e689ab59b6940a1a3fe7a317284fa97 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/1e689ab59b6940a1a3fe7a317284fa97 2024-11-25T12:59:21,174 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): de45cbd8ecc09a85a2c9aae45053021b#A#compaction#319 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:59:21,175 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/5c4e2348fcf540fba3e6f3c84df71525 is 175, key is test_row_0/A:col10/1732539558653/Put/seqid=0 2024-11-25T12:59:21,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742213_1389 (size=31617) 2024-11-25T12:59:21,188 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de45cbd8ecc09a85a2c9aae45053021b/C of de45cbd8ecc09a85a2c9aae45053021b into 1e689ab59b6940a1a3fe7a317284fa97(size=12.4 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:59:21,188 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de45cbd8ecc09a85a2c9aae45053021b: 2024-11-25T12:59:21,188 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., storeName=de45cbd8ecc09a85a2c9aae45053021b/C, priority=13, startTime=1732539560671; duration=0sec 2024-11-25T12:59:21,188 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-25T12:59:21,188 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de45cbd8ecc09a85a2c9aae45053021b:C 2024-11-25T12:59:21,189 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 3 compacting, 1 eligible, 16 blocking 2024-11-25T12:59:21,189 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-25T12:59:21,189 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-25T12:59:21,189 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. because compaction request was cancelled 2024-11-25T12:59:21,189 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de45cbd8ecc09a85a2c9aae45053021b:A 2024-11-25T12:59:21,189 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de45cbd8ecc09a85a2c9aae45053021b:C 2024-11-25T12:59:21,190 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:59:21,192 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:59:21,192 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): de45cbd8ecc09a85a2c9aae45053021b/B is initiating minor compaction (all files) 2024-11-25T12:59:21,192 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de45cbd8ecc09a85a2c9aae45053021b/B in TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:21,193 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/932c28f3872d4636a97c50e4b091c3cc, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/993d67a154834ce08fe73b31657910ca, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/eac9cd5f0256494ea803f9fdd86f3103, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/7dc7b6cf2fe140acb223a019be6e470e] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp, totalSize=47.9 K 2024-11-25T12:59:21,193 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 932c28f3872d4636a97c50e4b091c3cc, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732539557209 2024-11-25T12:59:21,193 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 993d67a154834ce08fe73b31657910ca, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732539557913 2024-11-25T12:59:21,193 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting eac9cd5f0256494ea803f9fdd86f3103, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732539558640 2024-11-25T12:59:21,194 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 7dc7b6cf2fe140acb223a019be6e470e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1732539560042 2024-11-25T12:59:21,201 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): de45cbd8ecc09a85a2c9aae45053021b#B#compaction#324 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:59:21,202 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/B/2ff4d8e14db04c358ca53f0f53a1845b is 50, key is test_row_0/B:col10/1732539560042/Put/seqid=0 2024-11-25T12:59:21,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:21,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46484 deadline: 1732539621220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:21,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:21,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:21,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:21,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:21,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46542 deadline: 1732539621221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:21,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:21,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46538 deadline: 1732539621226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:21,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:21,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:21,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:21,234 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:21,234 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:21,234 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:21,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742214_1390 (size=12697) 2024-11-25T12:59:21,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-25T12:59:21,300 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:59:21,301 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:59:21,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-25T12:59:21,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-25T12:59:21,306 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:59:21,307 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:59:21,307 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:59:21,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:21,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46538 deadline: 1732539621343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:21,361 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:21,361 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:21,361 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:21,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:21,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46484 deadline: 1732539621343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:21,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:21,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46542 deadline: 1732539621345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:21,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:21,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:21,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:21,363 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:21,363 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:21,363 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:21,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-25T12:59:21,461 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-25T12:59:21,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:21,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. as already flushing 2024-11-25T12:59:21,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:21,462 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:21,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:21,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:21,553 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:21,566 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112586974719299c48bd94d8c16fdf840002_de45cbd8ecc09a85a2c9aae45053021b to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112586974719299c48bd94d8c16fdf840002_de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:21,567 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/69abf87c4b5045d2bd8c1614cc4fade0, store: [table=TestAcidGuarantees family=A region=de45cbd8ecc09a85a2c9aae45053021b] 2024-11-25T12:59:21,567 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/69abf87c4b5045d2bd8c1614cc4fade0 is 175, key is test_row_0/A:col10/1732539560767/Put/seqid=0 2024-11-25T12:59:21,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:21,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46542 deadline: 1732539621575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:21,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:21,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:21,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:21,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:21,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46484 deadline: 1732539621575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:21,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:21,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:21,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:21,588 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/5c4e2348fcf540fba3e6f3c84df71525 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/5c4e2348fcf540fba3e6f3c84df71525 2024-11-25T12:59:21,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:21,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46538 deadline: 1732539621575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:21,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:21,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:21,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:21,592 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de45cbd8ecc09a85a2c9aae45053021b/A of de45cbd8ecc09a85a2c9aae45053021b into 5c4e2348fcf540fba3e6f3c84df71525(size=30.9 K), total size for store is 61.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:59:21,593 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de45cbd8ecc09a85a2c9aae45053021b: 2024-11-25T12:59:21,593 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., storeName=de45cbd8ecc09a85a2c9aae45053021b/A, priority=13, startTime=1732539560671; duration=0sec 2024-11-25T12:59:21,593 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-25T12:59:21,593 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de45cbd8ecc09a85a2c9aae45053021b:A 2024-11-25T12:59:21,593 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 4 compacting, 0 eligible, 16 blocking 2024-11-25T12:59:21,593 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-25T12:59:21,593 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-25T12:59:21,593 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. because compaction request was cancelled 2024-11-25T12:59:21,593 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de45cbd8ecc09a85a2c9aae45053021b:B 2024-11-25T12:59:21,593 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-25T12:59:21,593 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-25T12:59:21,594 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-25T12:59:21,594 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. because compaction request was cancelled 2024-11-25T12:59:21,594 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de45cbd8ecc09a85a2c9aae45053021b:C 2024-11-25T12:59:21,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742215_1391 (size=39749) 2024-11-25T12:59:21,595 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=253, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/69abf87c4b5045d2bd8c1614cc4fade0 2024-11-25T12:59:21,601 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/B/f10ac63768fc49c9af1b68e3b5eabf13 is 50, key is test_row_0/B:col10/1732539560767/Put/seqid=0 2024-11-25T12:59:21,619 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-25T12:59:21,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:21,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. as already flushing 2024-11-25T12:59:21,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:21,620 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:21,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:21,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:21,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-25T12:59:21,642 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/B/2ff4d8e14db04c358ca53f0f53a1845b as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/2ff4d8e14db04c358ca53f0f53a1845b 2024-11-25T12:59:21,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742216_1392 (size=12151) 2024-11-25T12:59:21,662 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in de45cbd8ecc09a85a2c9aae45053021b/B of de45cbd8ecc09a85a2c9aae45053021b into 2ff4d8e14db04c358ca53f0f53a1845b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:59:21,662 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de45cbd8ecc09a85a2c9aae45053021b: 2024-11-25T12:59:21,663 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., storeName=de45cbd8ecc09a85a2c9aae45053021b/B, priority=12, startTime=1732539560982; duration=0sec 2024-11-25T12:59:21,663 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:59:21,663 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de45cbd8ecc09a85a2c9aae45053021b:B 2024-11-25T12:59:21,773 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-25T12:59:21,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:21,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. as already flushing 2024-11-25T12:59:21,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:21,773 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:21,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:21,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:21,901 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:21,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46542 deadline: 1732539621891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:21,902 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:21,902 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:21,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:21,902 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:21,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46484 deadline: 1732539621891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:21,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:21,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:21,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:21,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:21,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46538 deadline: 1732539621902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:21,923 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:21,923 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:21,923 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:21,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-25T12:59:21,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:21,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. as already flushing 2024-11-25T12:59:21,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:21,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-25T12:59:21,930 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:21,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:21,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:22,057 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/B/f10ac63768fc49c9af1b68e3b5eabf13 2024-11-25T12:59:22,077 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/C/100acd5d34c04f5f8ed65c205de40ad6 is 50, key is test_row_0/C:col10/1732539560767/Put/seqid=0 2024-11-25T12:59:22,085 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-25T12:59:22,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:22,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. as already flushing 2024-11-25T12:59:22,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:22,085 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:22,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:22,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:22,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742217_1393 (size=12151) 2024-11-25T12:59:22,133 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/C/100acd5d34c04f5f8ed65c205de40ad6 2024-11-25T12:59:22,139 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/69abf87c4b5045d2bd8c1614cc4fade0 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/69abf87c4b5045d2bd8c1614cc4fade0 2024-11-25T12:59:22,145 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/69abf87c4b5045d2bd8c1614cc4fade0, entries=200, sequenceid=253, filesize=38.8 K 2024-11-25T12:59:22,146 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/B/f10ac63768fc49c9af1b68e3b5eabf13 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/f10ac63768fc49c9af1b68e3b5eabf13 2024-11-25T12:59:22,151 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/f10ac63768fc49c9af1b68e3b5eabf13, entries=150, sequenceid=253, filesize=11.9 K 2024-11-25T12:59:22,152 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/C/100acd5d34c04f5f8ed65c205de40ad6 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/100acd5d34c04f5f8ed65c205de40ad6 2024-11-25T12:59:22,158 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/100acd5d34c04f5f8ed65c205de40ad6, entries=150, sequenceid=253, filesize=11.9 K 2024-11-25T12:59:22,159 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for de45cbd8ecc09a85a2c9aae45053021b in 1044ms, sequenceid=253, compaction requested=true 2024-11-25T12:59:22,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for de45cbd8ecc09a85a2c9aae45053021b: 2024-11-25T12:59:22,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de45cbd8ecc09a85a2c9aae45053021b:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:59:22,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:59:22,159 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:59:22,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de45cbd8ecc09a85a2c9aae45053021b:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:59:22,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:59:22,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de45cbd8ecc09a85a2c9aae45053021b:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:59:22,159 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-25T12:59:22,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:59:22,161 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-25T12:59:22,161 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-25T12:59:22,161 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. because compaction request was cancelled 2024-11-25T12:59:22,161 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de45cbd8ecc09a85a2c9aae45053021b:B 2024-11-25T12:59:22,161 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:59:22,162 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102471 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:59:22,162 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1541): de45cbd8ecc09a85a2c9aae45053021b/A is initiating minor compaction (all files) 2024-11-25T12:59:22,162 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de45cbd8ecc09a85a2c9aae45053021b/A in TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:22,162 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/5c4e2348fcf540fba3e6f3c84df71525, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/1cab360fd6bd4cf9a9b6cdb95e1f750e, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/69abf87c4b5045d2bd8c1614cc4fade0] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp, totalSize=100.1 K 2024-11-25T12:59:22,162 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:22,162 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. files: [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/5c4e2348fcf540fba3e6f3c84df71525, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/1cab360fd6bd4cf9a9b6cdb95e1f750e, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/69abf87c4b5045d2bd8c1614cc4fade0] 2024-11-25T12:59:22,162 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c4e2348fcf540fba3e6f3c84df71525, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732539558640 2024-11-25T12:59:22,163 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1cab360fd6bd4cf9a9b6cdb95e1f750e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1732539560042 2024-11-25T12:59:22,163 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:59:22,163 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): de45cbd8ecc09a85a2c9aae45053021b/C is initiating minor compaction (all files) 2024-11-25T12:59:22,163 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de45cbd8ecc09a85a2c9aae45053021b/C in TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:22,163 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/1e689ab59b6940a1a3fe7a317284fa97, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/143f1559c61540bf935b24eba0dce8ed, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/100acd5d34c04f5f8ed65c205de40ad6] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp, totalSize=36.1 K 2024-11-25T12:59:22,163 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69abf87c4b5045d2bd8c1614cc4fade0, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732539560764 2024-11-25T12:59:22,164 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e689ab59b6940a1a3fe7a317284fa97, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732539558640 2024-11-25T12:59:22,164 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 143f1559c61540bf935b24eba0dce8ed, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1732539560042 2024-11-25T12:59:22,165 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 100acd5d34c04f5f8ed65c205de40ad6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732539560764 2024-11-25T12:59:22,180 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=de45cbd8ecc09a85a2c9aae45053021b] 2024-11-25T12:59:22,184 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): de45cbd8ecc09a85a2c9aae45053021b#C#compaction#328 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:59:22,184 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/C/97c7a80fdc1a43a3971af7fc8bcb1389 is 50, key is test_row_0/C:col10/1732539560767/Put/seqid=0 2024-11-25T12:59:22,196 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112502241a64448141bf97b602435081555d_de45cbd8ecc09a85a2c9aae45053021b store=[table=TestAcidGuarantees family=A region=de45cbd8ecc09a85a2c9aae45053021b] 2024-11-25T12:59:22,198 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112502241a64448141bf97b602435081555d_de45cbd8ecc09a85a2c9aae45053021b, store=[table=TestAcidGuarantees family=A region=de45cbd8ecc09a85a2c9aae45053021b] 2024-11-25T12:59:22,198 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112502241a64448141bf97b602435081555d_de45cbd8ecc09a85a2c9aae45053021b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=de45cbd8ecc09a85a2c9aae45053021b] 2024-11-25T12:59:22,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742218_1394 (size=12765) 2024-11-25T12:59:22,222 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/C/97c7a80fdc1a43a3971af7fc8bcb1389 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/97c7a80fdc1a43a3971af7fc8bcb1389 2024-11-25T12:59:22,238 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-25T12:59:22,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:22,239 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de45cbd8ecc09a85a2c9aae45053021b/C of de45cbd8ecc09a85a2c9aae45053021b into 97c7a80fdc1a43a3971af7fc8bcb1389(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:59:22,239 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing de45cbd8ecc09a85a2c9aae45053021b 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-25T12:59:22,239 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de45cbd8ecc09a85a2c9aae45053021b: 2024-11-25T12:59:22,239 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., storeName=de45cbd8ecc09a85a2c9aae45053021b/C, priority=13, startTime=1732539562159; duration=0sec 2024-11-25T12:59:22,239 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:59:22,239 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de45cbd8ecc09a85a2c9aae45053021b:C 2024-11-25T12:59:22,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742219_1395 (size=4469) 2024-11-25T12:59:22,253 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): de45cbd8ecc09a85a2c9aae45053021b#A#compaction#327 average throughput is 0.33 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:59:22,253 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/f580e606fc194e4aa48d6f597086e780 is 175, key is test_row_0/A:col10/1732539560767/Put/seqid=0 2024-11-25T12:59:22,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411251ce71830dbe541008fb0f15fbf42de62_de45cbd8ecc09a85a2c9aae45053021b is 50, key is test_row_0/A:col10/1732539561219/Put/seqid=0 2024-11-25T12:59:22,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742220_1396 (size=31719) 2024-11-25T12:59:22,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742221_1397 (size=12454) 2024-11-25T12:59:22,417 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. as already flushing 2024-11-25T12:59:22,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:22,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-25T12:59:22,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:22,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46542 deadline: 1732539622521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:22,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:22,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:22,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:22,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:22,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46538 deadline: 1732539622528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:22,545 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:22,545 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:22,545 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:22,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:22,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46484 deadline: 1732539622531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:22,546 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:22,546 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:22,546 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:22,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:22,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46538 deadline: 1732539622652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:22,661 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:22,661 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:22,661 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:22,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:22,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46542 deadline: 1732539622660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:22,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:22,668 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:22,668 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:22,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46484 deadline: 1732539622660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:22,668 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:22,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:22,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:22,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:22,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:22,703 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411251ce71830dbe541008fb0f15fbf42de62_de45cbd8ecc09a85a2c9aae45053021b to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411251ce71830dbe541008fb0f15fbf42de62_de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:22,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/011acaad059b4479805555e14887db3b, store: [table=TestAcidGuarantees family=A region=de45cbd8ecc09a85a2c9aae45053021b] 2024-11-25T12:59:22,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/011acaad059b4479805555e14887db3b is 175, key is test_row_0/A:col10/1732539561219/Put/seqid=0 2024-11-25T12:59:22,712 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/f580e606fc194e4aa48d6f597086e780 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/f580e606fc194e4aa48d6f597086e780 2024-11-25T12:59:22,722 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de45cbd8ecc09a85a2c9aae45053021b/A of de45cbd8ecc09a85a2c9aae45053021b into f580e606fc194e4aa48d6f597086e780(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:59:22,722 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de45cbd8ecc09a85a2c9aae45053021b: 2024-11-25T12:59:22,722 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., storeName=de45cbd8ecc09a85a2c9aae45053021b/A, priority=13, startTime=1732539562159; duration=0sec 2024-11-25T12:59:22,722 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:59:22,722 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de45cbd8ecc09a85a2c9aae45053021b:A 2024-11-25T12:59:22,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742222_1398 (size=31255) 2024-11-25T12:59:22,762 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=279, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/011acaad059b4479805555e14887db3b 2024-11-25T12:59:22,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/B/1bde5ed595f04979ba1e83e7950922ea is 50, key is test_row_0/B:col10/1732539561219/Put/seqid=0 2024-11-25T12:59:22,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742223_1399 (size=12301) 2024-11-25T12:59:22,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:22,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46538 deadline: 1732539622871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:22,882 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:22,882 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:22,882 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:22,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:22,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46542 deadline: 1732539622880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:22,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:22,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46484 deadline: 1732539622881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:22,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:22,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:22,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:22,895 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:22,895 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:22,895 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:23,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:23,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46538 deadline: 1732539623190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:23,203 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:23,203 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:23,203 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:23,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:23,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46484 deadline: 1732539623212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:23,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:23,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:23,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:23,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:23,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46542 deadline: 1732539623215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:23,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:23,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:23,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:23,239 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/B/1bde5ed595f04979ba1e83e7950922ea 2024-11-25T12:59:23,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/C/f19a3f2872d44ca38d6848c07bbffa42 is 50, key is test_row_0/C:col10/1732539561219/Put/seqid=0 2024-11-25T12:59:23,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742224_1400 (size=12301) 2024-11-25T12:59:23,291 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/C/f19a3f2872d44ca38d6848c07bbffa42 2024-11-25T12:59:23,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/011acaad059b4479805555e14887db3b as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/011acaad059b4479805555e14887db3b 2024-11-25T12:59:23,333 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/011acaad059b4479805555e14887db3b, entries=150, sequenceid=279, filesize=30.5 K 2024-11-25T12:59:23,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/B/1bde5ed595f04979ba1e83e7950922ea as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/1bde5ed595f04979ba1e83e7950922ea 2024-11-25T12:59:23,339 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/1bde5ed595f04979ba1e83e7950922ea, entries=150, sequenceid=279, filesize=12.0 K 2024-11-25T12:59:23,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/C/f19a3f2872d44ca38d6848c07bbffa42 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/f19a3f2872d44ca38d6848c07bbffa42 2024-11-25T12:59:23,372 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/f19a3f2872d44ca38d6848c07bbffa42, entries=150, sequenceid=279, filesize=12.0 K 2024-11-25T12:59:23,373 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for de45cbd8ecc09a85a2c9aae45053021b in 1134ms, sequenceid=279, compaction requested=true 2024-11-25T12:59:23,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for de45cbd8ecc09a85a2c9aae45053021b: 2024-11-25T12:59:23,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:23,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-25T12:59:23,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-25T12:59:23,377 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-25T12:59:23,377 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0670 sec 2024-11-25T12:59:23,380 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 2.0770 sec 2024-11-25T12:59:23,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-25T12:59:23,467 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:59:23,476 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:59:23,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-25T12:59:23,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-25T12:59:23,480 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:59:23,481 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:59:23,481 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:59:23,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-25T12:59:23,638 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-25T12:59:23,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:23,639 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing de45cbd8ecc09a85a2c9aae45053021b 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-25T12:59:23,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125c002da0ff5a24c43ad5b8b62f189f67f_de45cbd8ecc09a85a2c9aae45053021b is 50, key is test_row_0/A:col10/1732539562456/Put/seqid=0 2024-11-25T12:59:23,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:23,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. as already flushing 2024-11-25T12:59:23,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742225_1401 (size=12454) 2024-11-25T12:59:23,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-25T12:59:23,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:23,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46542 deadline: 1732539623891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:23,908 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:23,908 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:23,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:23,908 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:23,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46484 deadline: 1732539623891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:23,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:23,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:23,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:23,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:23,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46538 deadline: 1732539623893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:23,910 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:23,910 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:23,910 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:24,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:24,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46542 deadline: 1732539624010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:24,019 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:24,019 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:24,019 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:24,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:24,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46484 deadline: 1732539624020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:24,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:24,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46538 deadline: 1732539624020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:24,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:24,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:24,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:24,032 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:24,032 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:24,032 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:24,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-25T12:59:24,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:24,129 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125c002da0ff5a24c43ad5b8b62f189f67f_de45cbd8ecc09a85a2c9aae45053021b to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125c002da0ff5a24c43ad5b8b62f189f67f_de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:24,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/bf57ead6c0af4031a4ab1e255a7ce99a, store: [table=TestAcidGuarantees family=A region=de45cbd8ecc09a85a2c9aae45053021b] 2024-11-25T12:59:24,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/bf57ead6c0af4031a4ab1e255a7ce99a is 175, key is test_row_0/A:col10/1732539562456/Put/seqid=0 2024-11-25T12:59:24,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742226_1402 (size=31255) 2024-11-25T12:59:24,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:24,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46542 deadline: 1732539624230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:24,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:24,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46484 deadline: 1732539624240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:24,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:24,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46538 deadline: 1732539624240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:24,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:24,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:24,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:24,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:24,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:24,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:24,258 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:24,258 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:24,258 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:24,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:24,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46484 deadline: 1732539624560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:24,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:24,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46538 deadline: 1732539624560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:24,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:24,578 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:24,578 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:24,578 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:24,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46542 deadline: 1732539624560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:24,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:24,579 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:24,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:24,579 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:24,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:24,579 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:24,582 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=295, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/bf57ead6c0af4031a4ab1e255a7ce99a 2024-11-25T12:59:24,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/B/88b1fc742cac4d8c8ee9a67a313b9b94 is 50, key is test_row_0/B:col10/1732539562456/Put/seqid=0 2024-11-25T12:59:24,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-25T12:59:24,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742227_1403 (size=12301) 2024-11-25T12:59:25,055 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/B/88b1fc742cac4d8c8ee9a67a313b9b94 2024-11-25T12:59:25,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/C/5bef005fccb24d4bbe133e6a055521b9 is 50, key is test_row_0/C:col10/1732539562456/Put/seqid=0 2024-11-25T12:59:25,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:25,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46538 deadline: 1732539625080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:25,099 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:25,099 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:25,099 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:25,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:25,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46484 deadline: 1732539625090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:25,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:25,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46542 deadline: 1732539625090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:25,111 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:25,111 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:25,112 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:25,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:25,113 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:25,113 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:25,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742228_1404 (size=12301) 2024-11-25T12:59:25,115 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/C/5bef005fccb24d4bbe133e6a055521b9 2024-11-25T12:59:25,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/bf57ead6c0af4031a4ab1e255a7ce99a as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/bf57ead6c0af4031a4ab1e255a7ce99a 2024-11-25T12:59:25,125 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/bf57ead6c0af4031a4ab1e255a7ce99a, entries=150, sequenceid=295, filesize=30.5 K 2024-11-25T12:59:25,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/B/88b1fc742cac4d8c8ee9a67a313b9b94 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/88b1fc742cac4d8c8ee9a67a313b9b94 2024-11-25T12:59:25,137 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/88b1fc742cac4d8c8ee9a67a313b9b94, entries=150, sequenceid=295, filesize=12.0 K 2024-11-25T12:59:25,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/C/5bef005fccb24d4bbe133e6a055521b9 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/5bef005fccb24d4bbe133e6a055521b9 2024-11-25T12:59:25,149 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/5bef005fccb24d4bbe133e6a055521b9, entries=150, sequenceid=295, filesize=12.0 K 2024-11-25T12:59:25,151 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for de45cbd8ecc09a85a2c9aae45053021b in 1511ms, sequenceid=295, compaction requested=true 2024-11-25T12:59:25,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for de45cbd8ecc09a85a2c9aae45053021b: 2024-11-25T12:59:25,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:25,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-25T12:59:25,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-25T12:59:25,161 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-25T12:59:25,161 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6770 sec 2024-11-25T12:59:25,172 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 1.6870 sec 2024-11-25T12:59:25,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-25T12:59:25,640 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:59:25,641 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:59:25,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-25T12:59:25,644 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:59:25,644 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:59:25,645 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:59:25,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-25T12:59:25,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-25T12:59:25,796 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-25T12:59:25,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:25,797 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing de45cbd8ecc09a85a2c9aae45053021b 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-25T12:59:25,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125dbae1b8b34d94255abb2f47d26f9a4f4_de45cbd8ecc09a85a2c9aae45053021b is 50, key is test_row_0/A:col10/1732539563891/Put/seqid=0 2024-11-25T12:59:25,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742229_1405 (size=12454) 2024-11-25T12:59:25,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-25T12:59:26,114 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. as already flushing 2024-11-25T12:59:26,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:26,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:26,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46538 deadline: 1732539626206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:26,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:26,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46484 deadline: 1732539626207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:26,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:26,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:26,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:26,224 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:26,224 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:26,224 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:26,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:26,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46542 deadline: 1732539626213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:26,227 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:26,227 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:26,227 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:26,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:26,252 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125dbae1b8b34d94255abb2f47d26f9a4f4_de45cbd8ecc09a85a2c9aae45053021b to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125dbae1b8b34d94255abb2f47d26f9a4f4_de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:26,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/158f909676604451ada3c58b6d1e7ab7, store: [table=TestAcidGuarantees family=A region=de45cbd8ecc09a85a2c9aae45053021b] 2024-11-25T12:59:26,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/158f909676604451ada3c58b6d1e7ab7 is 175, key is test_row_0/A:col10/1732539563891/Put/seqid=0 2024-11-25T12:59:26,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-25T12:59:26,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742230_1406 (size=31255) 2024-11-25T12:59:26,301 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=316, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/158f909676604451ada3c58b6d1e7ab7 2024-11-25T12:59:26,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/B/23a6e1eaff934ba8ac80a63278306fc6 is 50, key is test_row_0/B:col10/1732539563891/Put/seqid=0 2024-11-25T12:59:26,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742231_1407 (size=12301) 2024-11-25T12:59:26,323 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/B/23a6e1eaff934ba8ac80a63278306fc6 2024-11-25T12:59:26,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/C/6a8729a388474378b0d3654abe32ab97 is 50, key is test_row_0/C:col10/1732539563891/Put/seqid=0 2024-11-25T12:59:26,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742232_1408 (size=12301) 2024-11-25T12:59:26,346 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/C/6a8729a388474378b0d3654abe32ab97 2024-11-25T12:59:26,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:26,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46542 deadline: 1732539626331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:26,348 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:26,348 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:26,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:26,348 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:26,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46538 deadline: 1732539626331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:26,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:26,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:26,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:26,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:26,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46484 deadline: 1732539626330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:26,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:26,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:26,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:26,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/158f909676604451ada3c58b6d1e7ab7 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/158f909676604451ada3c58b6d1e7ab7 2024-11-25T12:59:26,367 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/158f909676604451ada3c58b6d1e7ab7, entries=150, sequenceid=316, filesize=30.5 K 2024-11-25T12:59:26,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/B/23a6e1eaff934ba8ac80a63278306fc6 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/23a6e1eaff934ba8ac80a63278306fc6 2024-11-25T12:59:26,372 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/23a6e1eaff934ba8ac80a63278306fc6, entries=150, sequenceid=316, filesize=12.0 K 2024-11-25T12:59:26,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/C/6a8729a388474378b0d3654abe32ab97 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/6a8729a388474378b0d3654abe32ab97 2024-11-25T12:59:26,379 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/6a8729a388474378b0d3654abe32ab97, entries=150, sequenceid=316, filesize=12.0 K 2024-11-25T12:59:26,380 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for de45cbd8ecc09a85a2c9aae45053021b in 584ms, sequenceid=316, compaction requested=true 2024-11-25T12:59:26,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for de45cbd8ecc09a85a2c9aae45053021b: 2024-11-25T12:59:26,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:26,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-25T12:59:26,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-25T12:59:26,383 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-25T12:59:26,383 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 737 msec 2024-11-25T12:59:26,384 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 742 msec 2024-11-25T12:59:26,571 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:59:26,571 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing de45cbd8ecc09a85a2c9aae45053021b 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-25T12:59:26,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:26,598 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112539590864c27547b6a944dcdb540b26b5_de45cbd8ecc09a85a2c9aae45053021b is 50, key is test_row_0/A:col10/1732539566211/Put/seqid=0 2024-11-25T12:59:26,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742233_1409 (size=14994) 2024-11-25T12:59:26,660 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:26,665 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112539590864c27547b6a944dcdb540b26b5_de45cbd8ecc09a85a2c9aae45053021b to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112539590864c27547b6a944dcdb540b26b5_de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:26,666 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/23cc4b4ce7bf47e7bfd1a1ba8da3b8c6, store: [table=TestAcidGuarantees family=A region=de45cbd8ecc09a85a2c9aae45053021b] 2024-11-25T12:59:26,667 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/23cc4b4ce7bf47e7bfd1a1ba8da3b8c6 is 175, key is test_row_0/A:col10/1732539566211/Put/seqid=0 2024-11-25T12:59:26,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:26,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46542 deadline: 1732539626701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:26,733 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:26,733 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:26,733 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:26,733 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:26,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46484 deadline: 1732539626704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:26,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:26,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:26,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:26,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:26,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46538 deadline: 1732539626710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:26,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742234_1410 (size=39949) 2024-11-25T12:59:26,737 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=332, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/23cc4b4ce7bf47e7bfd1a1ba8da3b8c6 2024-11-25T12:59:26,738 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:26,738 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:26,738 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:26,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-25T12:59:26,781 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:59:26,781 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/B/a21fd4644e584f82b40bce0e47bc8ac4 is 50, key is test_row_0/B:col10/1732539566211/Put/seqid=0 2024-11-25T12:59:26,782 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:59:26,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-25T12:59:26,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-25T12:59:26,784 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:59:26,785 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:59:26,785 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:59:26,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742235_1411 (size=12301) 2024-11-25T12:59:26,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:26,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46542 deadline: 1732539626842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:26,846 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:26,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:26,846 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:26,846 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:26,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46484 deadline: 1732539626842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:26,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:26,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:26,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:26,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:26,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46538 deadline: 1732539626845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:26,855 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:26,855 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:26,855 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:26,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-25T12:59:26,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-25T12:59:26,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:26,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. as already flushing 2024-11-25T12:59:26,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:26,941 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:26,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:26,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:27,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:27,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46542 deadline: 1732539627050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:27,062 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:27,062 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:27,062 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:27,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:27,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46484 deadline: 1732539627050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:27,062 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:27,062 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:27,062 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:27,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:27,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46538 deadline: 1732539627060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:27,071 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:27,071 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:27,071 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:27,096 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-25T12:59:27,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:27,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. as already flushing 2024-11-25T12:59:27,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:27,097 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:27,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:27,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:27,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-25T12:59:27,250 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/B/a21fd4644e584f82b40bce0e47bc8ac4 2024-11-25T12:59:27,252 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-25T12:59:27,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:27,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. as already flushing 2024-11-25T12:59:27,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:27,253 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:27,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:27,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:27,317 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/C/5eddcd3ef3c54ae585ab34880246d542 is 50, key is test_row_0/C:col10/1732539566211/Put/seqid=0 2024-11-25T12:59:27,355 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-25T12:59:27,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742236_1412 (size=12301) 2024-11-25T12:59:27,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:27,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46484 deadline: 1732539627375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:27,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:27,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:27,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:27,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:27,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46542 deadline: 1732539627376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:27,392 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:27,392 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:27,392 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:27,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:27,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46538 deadline: 1732539627380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:27,397 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:27,397 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:27,397 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:27,407 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-25T12:59:27,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:27,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. as already flushing 2024-11-25T12:59:27,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:27,408 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:27,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:27,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:27,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-25T12:59:27,560 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-25T12:59:27,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:27,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. as already flushing 2024-11-25T12:59:27,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:27,561 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:27,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:27,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:27,713 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-25T12:59:27,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:27,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. as already flushing 2024-11-25T12:59:27,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:27,717 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:27,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:27,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:27,777 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/C/5eddcd3ef3c54ae585ab34880246d542 2024-11-25T12:59:27,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/23cc4b4ce7bf47e7bfd1a1ba8da3b8c6 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/23cc4b4ce7bf47e7bfd1a1ba8da3b8c6 2024-11-25T12:59:27,786 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/23cc4b4ce7bf47e7bfd1a1ba8da3b8c6, entries=200, sequenceid=332, filesize=39.0 K 2024-11-25T12:59:27,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/B/a21fd4644e584f82b40bce0e47bc8ac4 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/a21fd4644e584f82b40bce0e47bc8ac4 2024-11-25T12:59:27,791 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/a21fd4644e584f82b40bce0e47bc8ac4, entries=150, sequenceid=332, filesize=12.0 K 2024-11-25T12:59:27,792 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/C/5eddcd3ef3c54ae585ab34880246d542 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/5eddcd3ef3c54ae585ab34880246d542 2024-11-25T12:59:27,801 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/5eddcd3ef3c54ae585ab34880246d542, entries=150, sequenceid=332, filesize=12.0 K 2024-11-25T12:59:27,802 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for de45cbd8ecc09a85a2c9aae45053021b in 1230ms, sequenceid=332, compaction requested=true 2024-11-25T12:59:27,802 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for de45cbd8ecc09a85a2c9aae45053021b: 2024-11-25T12:59:27,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de45cbd8ecc09a85a2c9aae45053021b:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:59:27,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:59:27,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de45cbd8ecc09a85a2c9aae45053021b:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:59:27,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:59:27,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de45cbd8ecc09a85a2c9aae45053021b:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:59:27,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-25T12:59:27,802 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-25T12:59:27,802 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-25T12:59:27,805 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 74052 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-25T12:59:27,805 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1541): de45cbd8ecc09a85a2c9aae45053021b/B is initiating minor compaction (all files) 2024-11-25T12:59:27,805 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de45cbd8ecc09a85a2c9aae45053021b/B in TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:27,805 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/2ff4d8e14db04c358ca53f0f53a1845b, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/f10ac63768fc49c9af1b68e3b5eabf13, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/1bde5ed595f04979ba1e83e7950922ea, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/88b1fc742cac4d8c8ee9a67a313b9b94, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/23a6e1eaff934ba8ac80a63278306fc6, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/a21fd4644e584f82b40bce0e47bc8ac4] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp, totalSize=72.3 K 2024-11-25T12:59:27,806 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 165433 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-25T12:59:27,806 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): de45cbd8ecc09a85a2c9aae45053021b/A is initiating minor compaction (all files) 2024-11-25T12:59:27,806 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de45cbd8ecc09a85a2c9aae45053021b/A in TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:27,806 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/f580e606fc194e4aa48d6f597086e780, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/011acaad059b4479805555e14887db3b, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/bf57ead6c0af4031a4ab1e255a7ce99a, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/158f909676604451ada3c58b6d1e7ab7, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/23cc4b4ce7bf47e7bfd1a1ba8da3b8c6] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp, totalSize=161.6 K 2024-11-25T12:59:27,806 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:27,806 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. files: [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/f580e606fc194e4aa48d6f597086e780, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/011acaad059b4479805555e14887db3b, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/bf57ead6c0af4031a4ab1e255a7ce99a, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/158f909676604451ada3c58b6d1e7ab7, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/23cc4b4ce7bf47e7bfd1a1ba8da3b8c6] 2024-11-25T12:59:27,807 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ff4d8e14db04c358ca53f0f53a1845b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1732539560042 2024-11-25T12:59:27,807 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting f580e606fc194e4aa48d6f597086e780, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732539560764 2024-11-25T12:59:27,807 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting f10ac63768fc49c9af1b68e3b5eabf13, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732539560764 2024-11-25T12:59:27,807 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 011acaad059b4479805555e14887db3b, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1732539561165 2024-11-25T12:59:27,808 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1bde5ed595f04979ba1e83e7950922ea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1732539561165 2024-11-25T12:59:27,808 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting bf57ead6c0af4031a4ab1e255a7ce99a, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732539562439 2024-11-25T12:59:27,808 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88b1fc742cac4d8c8ee9a67a313b9b94, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732539562439 2024-11-25T12:59:27,808 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 158f909676604451ada3c58b6d1e7ab7, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1732539563864 2024-11-25T12:59:27,808 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23a6e1eaff934ba8ac80a63278306fc6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1732539563864 2024-11-25T12:59:27,808 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 23cc4b4ce7bf47e7bfd1a1ba8da3b8c6, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1732539566183 2024-11-25T12:59:27,809 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting a21fd4644e584f82b40bce0e47bc8ac4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1732539566183 2024-11-25T12:59:27,834 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=de45cbd8ecc09a85a2c9aae45053021b] 2024-11-25T12:59:27,849 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): de45cbd8ecc09a85a2c9aae45053021b#B#compaction#342 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:59:27,849 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/B/0bd43695dd3b4ded82b53b18d1ae3e39 is 50, key is test_row_0/B:col10/1732539566211/Put/seqid=0 2024-11-25T12:59:27,859 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241125152a4c3572874d6eb5f00648289b618c_de45cbd8ecc09a85a2c9aae45053021b store=[table=TestAcidGuarantees family=A region=de45cbd8ecc09a85a2c9aae45053021b] 2024-11-25T12:59:27,862 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241125152a4c3572874d6eb5f00648289b618c_de45cbd8ecc09a85a2c9aae45053021b, store=[table=TestAcidGuarantees family=A region=de45cbd8ecc09a85a2c9aae45053021b] 2024-11-25T12:59:27,863 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125152a4c3572874d6eb5f00648289b618c_de45cbd8ecc09a85a2c9aae45053021b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=de45cbd8ecc09a85a2c9aae45053021b] 2024-11-25T12:59:27,893 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-25T12:59:27,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:27,893 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing de45cbd8ecc09a85a2c9aae45053021b 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-25T12:59:27,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742237_1413 (size=13051) 2024-11-25T12:59:27,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. as already flushing 2024-11-25T12:59:27,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-25T12:59:27,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:27,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742238_1414 (size=4469) 2024-11-25T12:59:27,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125162efc581e134a9ba60599eaad38801a_de45cbd8ecc09a85a2c9aae45053021b is 50, key is test_row_0/A:col10/1732539566669/Put/seqid=0 2024-11-25T12:59:27,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742239_1415 (size=12454) 2024-11-25T12:59:27,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:27,992 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125162efc581e134a9ba60599eaad38801a_de45cbd8ecc09a85a2c9aae45053021b to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125162efc581e134a9ba60599eaad38801a_de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:27,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/2f17c396668249ff83398a5fefdfdff9, store: [table=TestAcidGuarantees family=A region=de45cbd8ecc09a85a2c9aae45053021b] 2024-11-25T12:59:27,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/2f17c396668249ff83398a5fefdfdff9 is 175, key is test_row_0/A:col10/1732539566669/Put/seqid=0 2024-11-25T12:59:28,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742240_1416 (size=31255) 2024-11-25T12:59:28,010 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=352, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/2f17c396668249ff83398a5fefdfdff9 2024-11-25T12:59:28,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/B/b6d11b3100e3416f96b53f1a0e18e5c7 is 50, key is test_row_0/B:col10/1732539566669/Put/seqid=0 2024-11-25T12:59:28,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742241_1417 (size=12301) 2024-11-25T12:59:28,047 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/B/b6d11b3100e3416f96b53f1a0e18e5c7 2024-11-25T12:59:28,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/C/adada88cdd41440abcc8112c37adc0c7 is 50, key is test_row_0/C:col10/1732539566669/Put/seqid=0 2024-11-25T12:59:28,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:28,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46484 deadline: 1732539628063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:28,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742242_1418 (size=12301) 2024-11-25T12:59:28,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:28,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:28,089 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:28,089 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/C/adada88cdd41440abcc8112c37adc0c7 2024-11-25T12:59:28,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/2f17c396668249ff83398a5fefdfdff9 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/2f17c396668249ff83398a5fefdfdff9 2024-11-25T12:59:28,099 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/2f17c396668249ff83398a5fefdfdff9, entries=150, sequenceid=352, filesize=30.5 K 2024-11-25T12:59:28,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/B/b6d11b3100e3416f96b53f1a0e18e5c7 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/b6d11b3100e3416f96b53f1a0e18e5c7 2024-11-25T12:59:28,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:28,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46538 deadline: 1732539628066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:28,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:28,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46542 deadline: 1732539628066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:28,109 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:28,109 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:28,109 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:28,112 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/b6d11b3100e3416f96b53f1a0e18e5c7, entries=150, sequenceid=352, filesize=12.0 K 2024-11-25T12:59:28,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/C/adada88cdd41440abcc8112c37adc0c7 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/adada88cdd41440abcc8112c37adc0c7 2024-11-25T12:59:28,121 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/adada88cdd41440abcc8112c37adc0c7, entries=150, sequenceid=352, filesize=12.0 K 2024-11-25T12:59:28,122 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:28,122 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:28,122 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:28,122 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=100.63 KB/103050 for de45cbd8ecc09a85a2c9aae45053021b in 229ms, sequenceid=352, compaction requested=true 2024-11-25T12:59:28,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for de45cbd8ecc09a85a2c9aae45053021b: 2024-11-25T12:59:28,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:28,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-25T12:59:28,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-25T12:59:28,126 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-25T12:59:28,126 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3390 sec 2024-11-25T12:59:28,128 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 1.3450 sec 2024-11-25T12:59:28,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:28,196 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:59:28,196 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing de45cbd8ecc09a85a2c9aae45053021b 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-25T12:59:28,211 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411256e009a06099f45bebf9cb68db6f514d4_de45cbd8ecc09a85a2c9aae45053021b is 50, key is test_row_0/A:col10/1732539567983/Put/seqid=0 2024-11-25T12:59:28,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742243_1419 (size=14994) 2024-11-25T12:59:28,233 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:28,240 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411256e009a06099f45bebf9cb68db6f514d4_de45cbd8ecc09a85a2c9aae45053021b to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411256e009a06099f45bebf9cb68db6f514d4_de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:28,242 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/039203198ed04206929337fe56ea3aa2, store: [table=TestAcidGuarantees family=A region=de45cbd8ecc09a85a2c9aae45053021b] 2024-11-25T12:59:28,243 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/039203198ed04206929337fe56ea3aa2 is 175, key is test_row_0/A:col10/1732539567983/Put/seqid=0 2024-11-25T12:59:28,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742244_1420 (size=39949) 2024-11-25T12:59:28,261 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=371, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/039203198ed04206929337fe56ea3aa2 2024-11-25T12:59:28,270 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/B/ed9fe12708a748fb926dc21f21462dae is 50, key is test_row_0/B:col10/1732539567983/Put/seqid=0 2024-11-25T12:59:28,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742245_1421 (size=12301) 2024-11-25T12:59:28,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:28,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46484 deadline: 1732539628293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:28,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:28,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:28,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:28,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:28,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46538 deadline: 1732539628293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:28,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:28,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:28,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:28,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:28,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46542 deadline: 1732539628295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:28,310 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:28,310 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:28,310 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:28,313 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/B/0bd43695dd3b4ded82b53b18d1ae3e39 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/0bd43695dd3b4ded82b53b18d1ae3e39 2024-11-25T12:59:28,322 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in de45cbd8ecc09a85a2c9aae45053021b/B of de45cbd8ecc09a85a2c9aae45053021b into 0bd43695dd3b4ded82b53b18d1ae3e39(size=12.7 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:59:28,322 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de45cbd8ecc09a85a2c9aae45053021b: 2024-11-25T12:59:28,322 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., storeName=de45cbd8ecc09a85a2c9aae45053021b/B, priority=10, startTime=1732539567802; duration=0sec 2024-11-25T12:59:28,323 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:59:28,323 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de45cbd8ecc09a85a2c9aae45053021b:B 2024-11-25T12:59:28,323 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-25T12:59:28,324 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 74270 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-25T12:59:28,324 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1541): de45cbd8ecc09a85a2c9aae45053021b/C is initiating minor compaction (all files) 2024-11-25T12:59:28,324 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de45cbd8ecc09a85a2c9aae45053021b/C in TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:28,324 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/97c7a80fdc1a43a3971af7fc8bcb1389, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/f19a3f2872d44ca38d6848c07bbffa42, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/5bef005fccb24d4bbe133e6a055521b9, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/6a8729a388474378b0d3654abe32ab97, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/5eddcd3ef3c54ae585ab34880246d542, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/adada88cdd41440abcc8112c37adc0c7] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp, totalSize=72.5 K 2024-11-25T12:59:28,325 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97c7a80fdc1a43a3971af7fc8bcb1389, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732539560764 2024-11-25T12:59:28,325 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting f19a3f2872d44ca38d6848c07bbffa42, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1732539561165 2024-11-25T12:59:28,325 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5bef005fccb24d4bbe133e6a055521b9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732539562439 2024-11-25T12:59:28,325 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a8729a388474378b0d3654abe32ab97, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1732539563864 2024-11-25T12:59:28,325 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5eddcd3ef3c54ae585ab34880246d542, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1732539566183 2024-11-25T12:59:28,326 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting adada88cdd41440abcc8112c37adc0c7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1732539566669 2024-11-25T12:59:28,331 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): de45cbd8ecc09a85a2c9aae45053021b#A#compaction#341 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:59:28,332 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/2f7fb71f312f4170b0d711b3b56b6520 is 175, key is test_row_0/A:col10/1732539566211/Put/seqid=0 2024-11-25T12:59:28,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742246_1422 (size=32039) 2024-11-25T12:59:28,345 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): de45cbd8ecc09a85a2c9aae45053021b#C#compaction#348 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:59:28,345 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/C/2f31b946fd3243e2b6f7990173c590e5 is 50, key is test_row_0/C:col10/1732539566669/Put/seqid=0 2024-11-25T12:59:28,350 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/2f7fb71f312f4170b0d711b3b56b6520 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/2f7fb71f312f4170b0d711b3b56b6520 2024-11-25T12:59:28,354 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in de45cbd8ecc09a85a2c9aae45053021b/A of de45cbd8ecc09a85a2c9aae45053021b into 2f7fb71f312f4170b0d711b3b56b6520(size=31.3 K), total size for store is 61.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:59:28,354 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de45cbd8ecc09a85a2c9aae45053021b: 2024-11-25T12:59:28,354 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., storeName=de45cbd8ecc09a85a2c9aae45053021b/A, priority=11, startTime=1732539567802; duration=0sec 2024-11-25T12:59:28,354 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:59:28,354 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de45cbd8ecc09a85a2c9aae45053021b:A 2024-11-25T12:59:28,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742247_1423 (size=13119) 2024-11-25T12:59:28,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:28,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46484 deadline: 1732539628409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:28,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:28,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:28,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:28,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:28,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46542 deadline: 1732539628420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:28,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:28,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46538 deadline: 1732539628420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:28,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:28,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:28,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:28,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:28,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:28,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:28,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:28,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46538 deadline: 1732539628630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:28,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:28,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46484 deadline: 1732539628631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:28,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:28,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:28,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:28,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:28,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:28,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:28,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:28,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46542 deadline: 1732539628640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:28,648 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:28,648 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:28,648 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:59:28,692 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/B/ed9fe12708a748fb926dc21f21462dae 2024-11-25T12:59:28,726 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/C/437b3ced73424ae7999edca4385667e3 is 50, key is test_row_0/C:col10/1732539567983/Put/seqid=0 2024-11-25T12:59:28,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742248_1424 (size=12301) 2024-11-25T12:59:28,751 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/C/437b3ced73424ae7999edca4385667e3 2024-11-25T12:59:28,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/039203198ed04206929337fe56ea3aa2 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/039203198ed04206929337fe56ea3aa2 2024-11-25T12:59:28,767 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/039203198ed04206929337fe56ea3aa2, entries=200, sequenceid=371, filesize=39.0 K 2024-11-25T12:59:28,771 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/B/ed9fe12708a748fb926dc21f21462dae as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/ed9fe12708a748fb926dc21f21462dae 2024-11-25T12:59:28,784 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/ed9fe12708a748fb926dc21f21462dae, entries=150, sequenceid=371, filesize=12.0 K 2024-11-25T12:59:28,784 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/C/2f31b946fd3243e2b6f7990173c590e5 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/2f31b946fd3243e2b6f7990173c590e5 2024-11-25T12:59:28,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/C/437b3ced73424ae7999edca4385667e3 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/437b3ced73424ae7999edca4385667e3 2024-11-25T12:59:28,794 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/437b3ced73424ae7999edca4385667e3, entries=150, sequenceid=371, filesize=12.0 K 2024-11-25T12:59:28,799 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for de45cbd8ecc09a85a2c9aae45053021b in 602ms, sequenceid=371, compaction requested=true 2024-11-25T12:59:28,799 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for de45cbd8ecc09a85a2c9aae45053021b: 2024-11-25T12:59:28,799 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in de45cbd8ecc09a85a2c9aae45053021b/C of de45cbd8ecc09a85a2c9aae45053021b into 2f31b946fd3243e2b6f7990173c590e5(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:59:28,799 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de45cbd8ecc09a85a2c9aae45053021b: 2024-11-25T12:59:28,799 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., storeName=de45cbd8ecc09a85a2c9aae45053021b/C, priority=10, startTime=1732539567802; duration=0sec 2024-11-25T12:59:28,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de45cbd8ecc09a85a2c9aae45053021b:A, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:59:28,799 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:59:28,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:59:28,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de45cbd8ecc09a85a2c9aae45053021b:B, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:59:28,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:59:28,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de45cbd8ecc09a85a2c9aae45053021b:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:59:28,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-25T12:59:28,799 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-25T12:59:28,799 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de45cbd8ecc09a85a2c9aae45053021b:C 2024-11-25T12:59:28,799 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:59:28,800 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103243 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:59:28,800 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): de45cbd8ecc09a85a2c9aae45053021b/A is initiating minor compaction (all files) 2024-11-25T12:59:28,800 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de45cbd8ecc09a85a2c9aae45053021b/A in TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:28,800 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/2f7fb71f312f4170b0d711b3b56b6520, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/2f17c396668249ff83398a5fefdfdff9, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/039203198ed04206929337fe56ea3aa2] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp, totalSize=100.8 K 2024-11-25T12:59:28,800 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:28,800 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. files: [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/2f7fb71f312f4170b0d711b3b56b6520, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/2f17c396668249ff83398a5fefdfdff9, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/039203198ed04206929337fe56ea3aa2] 2024-11-25T12:59:28,801 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f7fb71f312f4170b0d711b3b56b6520, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1732539566183 2024-11-25T12:59:28,801 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f17c396668249ff83398a5fefdfdff9, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1732539566669 2024-11-25T12:59:28,801 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 039203198ed04206929337fe56ea3aa2, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1732539567983 2024-11-25T12:59:28,802 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:59:28,802 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1541): de45cbd8ecc09a85a2c9aae45053021b/B is initiating minor compaction (all files) 2024-11-25T12:59:28,802 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de45cbd8ecc09a85a2c9aae45053021b/B in TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:28,802 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/0bd43695dd3b4ded82b53b18d1ae3e39, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/b6d11b3100e3416f96b53f1a0e18e5c7, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/ed9fe12708a748fb926dc21f21462dae] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp, totalSize=36.8 K 2024-11-25T12:59:28,803 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0bd43695dd3b4ded82b53b18d1ae3e39, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1732539566183 2024-11-25T12:59:28,804 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6d11b3100e3416f96b53f1a0e18e5c7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1732539566669 2024-11-25T12:59:28,805 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed9fe12708a748fb926dc21f21462dae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1732539567983 2024-11-25T12:59:28,808 INFO [Thread-1496 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1496. 2024-11-25T12:59:28,809 DEBUG [Thread-1496 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:59:28,809 DEBUG [Thread-1496 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:59:28,809 DEBUG [Thread-1496 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:59:28,809 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:59:28,814 INFO [Thread-1488 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1488. 2024-11-25T12:59:28,814 DEBUG [Thread-1488 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:59:28,814 DEBUG [Thread-1488 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:59:28,814 DEBUG [Thread-1488 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:59:28,814 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:59:28,816 INFO [Thread-1492 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1492. 2024-11-25T12:59:28,816 DEBUG [Thread-1492 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:59:28,816 DEBUG [Thread-1492 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:59:28,817 DEBUG [Thread-1492 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:59:28,817 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:59:28,820 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): de45cbd8ecc09a85a2c9aae45053021b#B#compaction#350 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:59:28,821 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/B/ed988dafef6f42dcbcb2465c2ae40594 is 50, key is test_row_0/B:col10/1732539567983/Put/seqid=0 2024-11-25T12:59:28,825 INFO [Thread-1490 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1490. 2024-11-25T12:59:28,825 DEBUG [Thread-1490 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:59:28,825 DEBUG [Thread-1490 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:59:28,825 DEBUG [Thread-1490 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:59:28,825 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:59:28,828 INFO [Thread-1494 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1494. 2024-11-25T12:59:28,828 DEBUG [Thread-1494 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:59:28,828 DEBUG [Thread-1494 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:59:28,828 DEBUG [Thread-1494 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:59:28,828 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:59:28,829 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=de45cbd8ecc09a85a2c9aae45053021b] 2024-11-25T12:59:28,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742249_1425 (size=13153) 2024-11-25T12:59:28,839 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112584a793c617ea4f26a1c75ca05e1651dc_de45cbd8ecc09a85a2c9aae45053021b store=[table=TestAcidGuarantees family=A region=de45cbd8ecc09a85a2c9aae45053021b] 2024-11-25T12:59:28,841 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112584a793c617ea4f26a1c75ca05e1651dc_de45cbd8ecc09a85a2c9aae45053021b, store=[table=TestAcidGuarantees family=A region=de45cbd8ecc09a85a2c9aae45053021b] 2024-11-25T12:59:28,842 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112584a793c617ea4f26a1c75ca05e1651dc_de45cbd8ecc09a85a2c9aae45053021b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=de45cbd8ecc09a85a2c9aae45053021b] 2024-11-25T12:59:28,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742250_1426 (size=4469) 2024-11-25T12:59:28,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-25T12:59:28,930 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:59:28,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:28,952 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:59:28,952 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing de45cbd8ecc09a85a2c9aae45053021b 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-25T12:59:28,953 INFO [Thread-1477 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1477. 2024-11-25T12:59:28,953 DEBUG [Thread-1477 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:59:28,953 DEBUG [Thread-1477 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:59:28,953 DEBUG [Thread-1477 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:59:28,953 INFO [Thread-1479 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1479. 2024-11-25T12:59:28,953 DEBUG [Thread-1479 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:59:28,953 DEBUG [Thread-1479 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:59:28,953 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:59:28,953 DEBUG [Thread-1479 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:59:28,953 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:59:28,954 INFO [Thread-1481 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1481. 2024-11-25T12:59:28,954 DEBUG [Thread-1481 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:59:28,954 DEBUG [Thread-1481 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:59:28,954 DEBUG [Thread-1481 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:59:28,954 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:59:28,966 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411250e2960f3fe414151a73874fbf98b5bbd_de45cbd8ecc09a85a2c9aae45053021b is 50, key is test_row_0/A:col10/1732539568951/Put/seqid=0 2024-11-25T12:59:28,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742251_1427 (size=12454) 2024-11-25T12:59:29,241 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/B/ed988dafef6f42dcbcb2465c2ae40594 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/ed988dafef6f42dcbcb2465c2ae40594 2024-11-25T12:59:29,245 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de45cbd8ecc09a85a2c9aae45053021b/B of de45cbd8ecc09a85a2c9aae45053021b into ed988dafef6f42dcbcb2465c2ae40594(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:59:29,245 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de45cbd8ecc09a85a2c9aae45053021b: 2024-11-25T12:59:29,245 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., storeName=de45cbd8ecc09a85a2c9aae45053021b/B, priority=13, startTime=1732539568799; duration=0sec 2024-11-25T12:59:29,245 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:59:29,245 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de45cbd8ecc09a85a2c9aae45053021b:B 2024-11-25T12:59:29,245 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-25T12:59:29,246 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-25T12:59:29,246 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-25T12:59:29,246 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. because compaction request was cancelled 2024-11-25T12:59:29,246 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de45cbd8ecc09a85a2c9aae45053021b:C 2024-11-25T12:59:29,250 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): de45cbd8ecc09a85a2c9aae45053021b#A#compaction#351 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:59:29,250 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/124ce5fa34ab48588aba02a57b55379f is 175, key is test_row_0/A:col10/1732539567983/Put/seqid=0 2024-11-25T12:59:29,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742252_1428 (size=32141) 2024-11-25T12:59:29,380 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:29,383 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411250e2960f3fe414151a73874fbf98b5bbd_de45cbd8ecc09a85a2c9aae45053021b to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411250e2960f3fe414151a73874fbf98b5bbd_de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:29,383 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/d711805a420343039ba6ea497882d935, store: [table=TestAcidGuarantees family=A region=de45cbd8ecc09a85a2c9aae45053021b] 2024-11-25T12:59:29,384 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/d711805a420343039ba6ea497882d935 is 175, key is test_row_0/A:col10/1732539568951/Put/seqid=0 2024-11-25T12:59:29,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742253_1429 (size=31255) 2024-11-25T12:59:29,660 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/124ce5fa34ab48588aba02a57b55379f as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/124ce5fa34ab48588aba02a57b55379f 2024-11-25T12:59:29,672 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de45cbd8ecc09a85a2c9aae45053021b/A of de45cbd8ecc09a85a2c9aae45053021b into 124ce5fa34ab48588aba02a57b55379f(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:59:29,672 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de45cbd8ecc09a85a2c9aae45053021b: 2024-11-25T12:59:29,672 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., storeName=de45cbd8ecc09a85a2c9aae45053021b/A, priority=13, startTime=1732539568799; duration=0sec 2024-11-25T12:59:29,672 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:59:29,672 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de45cbd8ecc09a85a2c9aae45053021b:A 2024-11-25T12:59:29,787 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=394, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/d711805a420343039ba6ea497882d935 2024-11-25T12:59:29,797 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/B/85532865d2bd4b1ab25e3acb1ab7f37f is 50, key is test_row_0/B:col10/1732539568951/Put/seqid=0 2024-11-25T12:59:29,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742254_1430 (size=12301) 2024-11-25T12:59:30,201 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/B/85532865d2bd4b1ab25e3acb1ab7f37f 2024-11-25T12:59:30,207 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/C/88fcd411f1ff4a5b950b32ba71239e91 is 50, key is test_row_0/C:col10/1732539568951/Put/seqid=0 2024-11-25T12:59:30,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742255_1431 (size=12301) 2024-11-25T12:59:30,610 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/C/88fcd411f1ff4a5b950b32ba71239e91 2024-11-25T12:59:30,626 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/d711805a420343039ba6ea497882d935 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/d711805a420343039ba6ea497882d935 2024-11-25T12:59:30,631 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/d711805a420343039ba6ea497882d935, entries=150, sequenceid=394, filesize=30.5 K 2024-11-25T12:59:30,632 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/B/85532865d2bd4b1ab25e3acb1ab7f37f as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/85532865d2bd4b1ab25e3acb1ab7f37f 2024-11-25T12:59:30,635 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/85532865d2bd4b1ab25e3acb1ab7f37f, entries=150, sequenceid=394, filesize=12.0 K 2024-11-25T12:59:30,636 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/C/88fcd411f1ff4a5b950b32ba71239e91 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/88fcd411f1ff4a5b950b32ba71239e91 2024-11-25T12:59:30,639 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/88fcd411f1ff4a5b950b32ba71239e91, entries=150, sequenceid=394, filesize=12.0 K 2024-11-25T12:59:30,640 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=0 B/0 for de45cbd8ecc09a85a2c9aae45053021b in 1688ms, sequenceid=394, compaction requested=true 2024-11-25T12:59:30,640 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for de45cbd8ecc09a85a2c9aae45053021b: 2024-11-25T12:59:30,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de45cbd8ecc09a85a2c9aae45053021b:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:59:30,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:59:30,640 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-25T12:59:30,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de45cbd8ecc09a85a2c9aae45053021b:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:59:30,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:59:30,640 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-25T12:59:30,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de45cbd8ecc09a85a2c9aae45053021b:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:59:30,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:59:30,641 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-25T12:59:30,641 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-25T12:59:30,641 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-25T12:59:30,641 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-25T12:59:30,641 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. because compaction request was cancelled 2024-11-25T12:59:30,641 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. because compaction request was cancelled 2024-11-25T12:59:30,641 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de45cbd8ecc09a85a2c9aae45053021b:A 2024-11-25T12:59:30,641 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de45cbd8ecc09a85a2c9aae45053021b:B 2024-11-25T12:59:30,641 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:59:30,641 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:59:30,641 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1541): de45cbd8ecc09a85a2c9aae45053021b/C is initiating minor compaction (all files) 2024-11-25T12:59:30,642 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de45cbd8ecc09a85a2c9aae45053021b/C in TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:30,642 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/2f31b946fd3243e2b6f7990173c590e5, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/437b3ced73424ae7999edca4385667e3, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/88fcd411f1ff4a5b950b32ba71239e91] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp, totalSize=36.8 K 2024-11-25T12:59:30,642 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f31b946fd3243e2b6f7990173c590e5, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1732539566669 2024-11-25T12:59:30,642 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 437b3ced73424ae7999edca4385667e3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1732539567983 2024-11-25T12:59:30,642 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88fcd411f1ff4a5b950b32ba71239e91, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1732539568291 2024-11-25T12:59:30,649 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): de45cbd8ecc09a85a2c9aae45053021b#C#compaction#355 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:59:30,649 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/C/a205b5e0b9bf4435abd7061e8f592b62 is 50, key is test_row_0/C:col10/1732539568951/Put/seqid=0 2024-11-25T12:59:30,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742256_1432 (size=13221) 2024-11-25T12:59:30,975 INFO [Thread-1485 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1485. 2024-11-25T12:59:30,975 DEBUG [Thread-1485 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:59:30,975 DEBUG [Thread-1485 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:59:30,975 DEBUG [Thread-1485 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:59:30,977 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:59:30,992 INFO [Thread-1483 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1483. 2024-11-25T12:59:30,992 DEBUG [Thread-1483 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:59:30,992 DEBUG [Thread-1483 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:59:30,992 DEBUG [Thread-1483 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:59:30,992 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:59:30,993 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-25T12:59:30,993 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 98 2024-11-25T12:59:30,993 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 88 2024-11-25T12:59:30,993 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 92 2024-11-25T12:59:30,993 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 22 2024-11-25T12:59:30,993 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 12 2024-11-25T12:59:30,993 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-25T12:59:30,993 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-25T12:59:30,993 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 847 2024-11-25T12:59:30,993 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2533 rows 2024-11-25T12:59:30,993 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 849 2024-11-25T12:59:30,993 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2539 rows 2024-11-25T12:59:30,993 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 837 2024-11-25T12:59:30,993 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2511 rows 2024-11-25T12:59:30,993 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 840 2024-11-25T12:59:30,993 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2520 rows 2024-11-25T12:59:30,993 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 851 2024-11-25T12:59:30,993 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2551 rows 2024-11-25T12:59:30,993 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T12:59:30,993 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T12:59:30,993 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:59:30,993 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:59:30,994 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:59:30,995 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-25T12:59:30,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-25T12:59:31,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-25T12:59:31,001 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732539571001"}]},"ts":"1732539571001"} 2024-11-25T12:59:31,002 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-25T12:59:31,002 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-25T12:59:31,003 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-25T12:59:31,004 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=de45cbd8ecc09a85a2c9aae45053021b, UNASSIGN}] 2024-11-25T12:59:31,005 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=de45cbd8ecc09a85a2c9aae45053021b, UNASSIGN 2024-11-25T12:59:31,005 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=105 updating hbase:meta row=de45cbd8ecc09a85a2c9aae45053021b, regionState=CLOSING, regionLocation=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:31,006 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46289 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=1ff8009e463e,35905,1732539449246, table=TestAcidGuarantees, region=de45cbd8ecc09a85a2c9aae45053021b. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-25T12:59:31,006 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=de45cbd8ecc09a85a2c9aae45053021b, UNASSIGN because future has completed 2024-11-25T12:59:31,007 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-25T12:59:31,007 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; CloseRegionProcedure de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246}] 2024-11-25T12:59:31,065 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/C/a205b5e0b9bf4435abd7061e8f592b62 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/a205b5e0b9bf4435abd7061e8f592b62 2024-11-25T12:59:31,073 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de45cbd8ecc09a85a2c9aae45053021b/C of de45cbd8ecc09a85a2c9aae45053021b into a205b5e0b9bf4435abd7061e8f592b62(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:59:31,073 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de45cbd8ecc09a85a2c9aae45053021b: 2024-11-25T12:59:31,073 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., storeName=de45cbd8ecc09a85a2c9aae45053021b/C, priority=13, startTime=1732539570640; duration=0sec 2024-11-25T12:59:31,073 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:59:31,073 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de45cbd8ecc09a85a2c9aae45053021b:C 2024-11-25T12:59:31,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-25T12:59:31,159 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] handler.UnassignRegionHandler(122): Close de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:31,159 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-25T12:59:31,159 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1722): Closing de45cbd8ecc09a85a2c9aae45053021b, disabling compactions & flushes 2024-11-25T12:59:31,159 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:31,159 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:31,159 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. after waiting 0 ms 2024-11-25T12:59:31,159 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:31,159 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(2902): Flushing de45cbd8ecc09a85a2c9aae45053021b 3/3 column families, dataSize=13.42 KB heapSize=35.91 KB 2024-11-25T12:59:31,167 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411253ca58c3dc7e842b0abb51bd9450558e0_de45cbd8ecc09a85a2c9aae45053021b is 50, key is test_row_1/A:col10/1732539570971/Put/seqid=0 2024-11-25T12:59:31,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742257_1433 (size=9914) 2024-11-25T12:59:31,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-25T12:59:31,593 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:31,607 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411253ca58c3dc7e842b0abb51bd9450558e0_de45cbd8ecc09a85a2c9aae45053021b to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411253ca58c3dc7e842b0abb51bd9450558e0_de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:31,615 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/b601ab3c545b4f5dafd6410936e19ae7, store: [table=TestAcidGuarantees family=A region=de45cbd8ecc09a85a2c9aae45053021b] 2024-11-25T12:59:31,616 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/b601ab3c545b4f5dafd6410936e19ae7 is 175, key is test_row_1/A:col10/1732539570971/Put/seqid=0 2024-11-25T12:59:31,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742258_1434 (size=22561) 2024-11-25T12:59:31,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-25T12:59:31,640 WARN [regionserver/1ff8009e463e:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 0 2024-11-25T12:59:31,642 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/6ba8a71ff08d4cd08b2f44d8330fd808, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/24af528763924d2293e8f9d80e391407, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/92d22b71473d4daf955c71692c758a90, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/b1e05ed9177e4b87972f9ae3f0c069d1, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/94baa5c4494549cf87acc14ea1a33dc9, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/97d59372c3164d30936aad8644d60849, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/fa0837cd25d244b89fdbe325a806773e, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/8c50ed3a57b34053874f94fed69e0b4b, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/b4f17e1a272f490a95e447470d29279f, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/a4934b14da9048d69dae045b8c44fee3, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/02639827a95e4c91a59192950db51610, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/e76487964ff84cc88388f533f8090a7c, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/5b28e5a0b9234bd9ba0db1b56e34e29a, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/17d9d87b463649bea96531fcce85fab0, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/fe9f657cb5f147aa80f33c2e4e2ec001, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/5c4e2348fcf540fba3e6f3c84df71525, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/1cab360fd6bd4cf9a9b6cdb95e1f750e, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/69abf87c4b5045d2bd8c1614cc4fade0, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/f580e606fc194e4aa48d6f597086e780, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/011acaad059b4479805555e14887db3b, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/bf57ead6c0af4031a4ab1e255a7ce99a, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/158f909676604451ada3c58b6d1e7ab7, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/23cc4b4ce7bf47e7bfd1a1ba8da3b8c6, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/2f7fb71f312f4170b0d711b3b56b6520, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/2f17c396668249ff83398a5fefdfdff9, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/039203198ed04206929337fe56ea3aa2] to archive 2024-11-25T12:59:31,659 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T12:59:31,668 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/6ba8a71ff08d4cd08b2f44d8330fd808 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/6ba8a71ff08d4cd08b2f44d8330fd808 2024-11-25T12:59:31,676 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/24af528763924d2293e8f9d80e391407 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/24af528763924d2293e8f9d80e391407 2024-11-25T12:59:31,678 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/92d22b71473d4daf955c71692c758a90 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/92d22b71473d4daf955c71692c758a90 2024-11-25T12:59:31,686 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/b1e05ed9177e4b87972f9ae3f0c069d1 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/b1e05ed9177e4b87972f9ae3f0c069d1 2024-11-25T12:59:31,691 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/94baa5c4494549cf87acc14ea1a33dc9 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/94baa5c4494549cf87acc14ea1a33dc9 2024-11-25T12:59:31,692 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/97d59372c3164d30936aad8644d60849 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/97d59372c3164d30936aad8644d60849 2024-11-25T12:59:31,694 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/fa0837cd25d244b89fdbe325a806773e to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/fa0837cd25d244b89fdbe325a806773e 2024-11-25T12:59:31,695 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/8c50ed3a57b34053874f94fed69e0b4b to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/8c50ed3a57b34053874f94fed69e0b4b 2024-11-25T12:59:31,701 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/b4f17e1a272f490a95e447470d29279f to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/b4f17e1a272f490a95e447470d29279f 2024-11-25T12:59:31,703 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/a4934b14da9048d69dae045b8c44fee3 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/a4934b14da9048d69dae045b8c44fee3 2024-11-25T12:59:31,704 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/02639827a95e4c91a59192950db51610 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/02639827a95e4c91a59192950db51610 2024-11-25T12:59:31,705 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/e76487964ff84cc88388f533f8090a7c to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/e76487964ff84cc88388f533f8090a7c 2024-11-25T12:59:31,707 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/5b28e5a0b9234bd9ba0db1b56e34e29a to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/5b28e5a0b9234bd9ba0db1b56e34e29a 2024-11-25T12:59:31,708 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/17d9d87b463649bea96531fcce85fab0 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/17d9d87b463649bea96531fcce85fab0 2024-11-25T12:59:31,708 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/fe9f657cb5f147aa80f33c2e4e2ec001 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/fe9f657cb5f147aa80f33c2e4e2ec001 2024-11-25T12:59:31,709 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/5c4e2348fcf540fba3e6f3c84df71525 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/5c4e2348fcf540fba3e6f3c84df71525 2024-11-25T12:59:31,710 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/1cab360fd6bd4cf9a9b6cdb95e1f750e to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/1cab360fd6bd4cf9a9b6cdb95e1f750e 2024-11-25T12:59:31,711 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/69abf87c4b5045d2bd8c1614cc4fade0 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/69abf87c4b5045d2bd8c1614cc4fade0 2024-11-25T12:59:31,712 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/f580e606fc194e4aa48d6f597086e780 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/f580e606fc194e4aa48d6f597086e780 2024-11-25T12:59:31,714 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/011acaad059b4479805555e14887db3b to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/011acaad059b4479805555e14887db3b 2024-11-25T12:59:31,715 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/bf57ead6c0af4031a4ab1e255a7ce99a to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/bf57ead6c0af4031a4ab1e255a7ce99a 2024-11-25T12:59:31,716 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/158f909676604451ada3c58b6d1e7ab7 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/158f909676604451ada3c58b6d1e7ab7 2024-11-25T12:59:31,717 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/23cc4b4ce7bf47e7bfd1a1ba8da3b8c6 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/23cc4b4ce7bf47e7bfd1a1ba8da3b8c6 2024-11-25T12:59:31,718 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/2f7fb71f312f4170b0d711b3b56b6520 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/2f7fb71f312f4170b0d711b3b56b6520 2024-11-25T12:59:31,719 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/2f17c396668249ff83398a5fefdfdff9 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/2f17c396668249ff83398a5fefdfdff9 2024-11-25T12:59:31,720 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/039203198ed04206929337fe56ea3aa2 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/039203198ed04206929337fe56ea3aa2 2024-11-25T12:59:31,724 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/dbfc859d33714e3a8c25f1e91081474d, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/bc614b2ddf3e4888a43bc57d5c99b1bb, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/668e0480eff441cb8abc21b4032cba27, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/1e6e10ae809c4a87885da5c22a214ff2, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/0db34574bfab49e3978f2f7d0120cfcc, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/a72d2d1a3be24c6c856d67f1fcab10c0, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/73817ec81d5f4eb19f2392140fb9f60b, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/4fe14c9677664c5bafb401850e2116e4, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/483b294026f34b5f8428b367f2ce7b8c, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/972f8bbfb6154d5dabb945182a608a2c, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/e6110a0b04d94e5eb794ff68941ecb85, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/932c28f3872d4636a97c50e4b091c3cc, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/251162e0708a4c62a83edf9ff307425d, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/993d67a154834ce08fe73b31657910ca, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/eac9cd5f0256494ea803f9fdd86f3103, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/2ff4d8e14db04c358ca53f0f53a1845b, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/7dc7b6cf2fe140acb223a019be6e470e, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/f10ac63768fc49c9af1b68e3b5eabf13, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/1bde5ed595f04979ba1e83e7950922ea, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/88b1fc742cac4d8c8ee9a67a313b9b94, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/23a6e1eaff934ba8ac80a63278306fc6, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/0bd43695dd3b4ded82b53b18d1ae3e39, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/a21fd4644e584f82b40bce0e47bc8ac4, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/b6d11b3100e3416f96b53f1a0e18e5c7, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/ed9fe12708a748fb926dc21f21462dae] to archive 2024-11-25T12:59:31,725 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T12:59:31,726 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/dbfc859d33714e3a8c25f1e91081474d to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/dbfc859d33714e3a8c25f1e91081474d 2024-11-25T12:59:31,727 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/bc614b2ddf3e4888a43bc57d5c99b1bb to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/bc614b2ddf3e4888a43bc57d5c99b1bb 2024-11-25T12:59:31,728 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/668e0480eff441cb8abc21b4032cba27 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/668e0480eff441cb8abc21b4032cba27 2024-11-25T12:59:31,729 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/1e6e10ae809c4a87885da5c22a214ff2 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/1e6e10ae809c4a87885da5c22a214ff2 2024-11-25T12:59:31,730 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/0db34574bfab49e3978f2f7d0120cfcc to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/0db34574bfab49e3978f2f7d0120cfcc 2024-11-25T12:59:31,731 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/a72d2d1a3be24c6c856d67f1fcab10c0 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/a72d2d1a3be24c6c856d67f1fcab10c0 2024-11-25T12:59:31,731 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/73817ec81d5f4eb19f2392140fb9f60b to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/73817ec81d5f4eb19f2392140fb9f60b 2024-11-25T12:59:31,732 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/4fe14c9677664c5bafb401850e2116e4 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/4fe14c9677664c5bafb401850e2116e4 2024-11-25T12:59:31,733 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/483b294026f34b5f8428b367f2ce7b8c to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/483b294026f34b5f8428b367f2ce7b8c 2024-11-25T12:59:31,734 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/972f8bbfb6154d5dabb945182a608a2c to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/972f8bbfb6154d5dabb945182a608a2c 2024-11-25T12:59:31,734 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/e6110a0b04d94e5eb794ff68941ecb85 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/e6110a0b04d94e5eb794ff68941ecb85 2024-11-25T12:59:31,735 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/932c28f3872d4636a97c50e4b091c3cc to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/932c28f3872d4636a97c50e4b091c3cc 2024-11-25T12:59:31,736 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/251162e0708a4c62a83edf9ff307425d to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/251162e0708a4c62a83edf9ff307425d 2024-11-25T12:59:31,736 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/993d67a154834ce08fe73b31657910ca to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/993d67a154834ce08fe73b31657910ca 2024-11-25T12:59:31,737 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/eac9cd5f0256494ea803f9fdd86f3103 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/eac9cd5f0256494ea803f9fdd86f3103 2024-11-25T12:59:31,737 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/2ff4d8e14db04c358ca53f0f53a1845b to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/2ff4d8e14db04c358ca53f0f53a1845b 2024-11-25T12:59:31,738 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/7dc7b6cf2fe140acb223a019be6e470e to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/7dc7b6cf2fe140acb223a019be6e470e 2024-11-25T12:59:31,739 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/f10ac63768fc49c9af1b68e3b5eabf13 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/f10ac63768fc49c9af1b68e3b5eabf13 2024-11-25T12:59:31,739 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/1bde5ed595f04979ba1e83e7950922ea to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/1bde5ed595f04979ba1e83e7950922ea 2024-11-25T12:59:31,740 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/88b1fc742cac4d8c8ee9a67a313b9b94 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/88b1fc742cac4d8c8ee9a67a313b9b94 2024-11-25T12:59:31,741 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/23a6e1eaff934ba8ac80a63278306fc6 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/23a6e1eaff934ba8ac80a63278306fc6 2024-11-25T12:59:31,741 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/0bd43695dd3b4ded82b53b18d1ae3e39 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/0bd43695dd3b4ded82b53b18d1ae3e39 2024-11-25T12:59:31,742 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/a21fd4644e584f82b40bce0e47bc8ac4 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/a21fd4644e584f82b40bce0e47bc8ac4 2024-11-25T12:59:31,743 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/b6d11b3100e3416f96b53f1a0e18e5c7 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/b6d11b3100e3416f96b53f1a0e18e5c7 2024-11-25T12:59:31,743 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/ed9fe12708a748fb926dc21f21462dae to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/ed9fe12708a748fb926dc21f21462dae 2024-11-25T12:59:31,746 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/26df2c6ab7a741e2ad34b54a06238814, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/075b4de41cba48a59dc9e7d580f7d67d, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/0ddd0ca5d2c040e68b48a0d397fb9cea, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/2b66783f288e4f9098d11afc71c24698, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/d810f965a4044ebfa0e724befc983c29, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/dc5a19e2d9fa4ced956635a857081496, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/e9dfe42e7c76473282927aa94baffaee, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/da9cb6997f47465088b16f58ad38078a, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/c722365822cc4185a1742d01698b2b1f, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/20a29e5a1b954d66bfb20df6cc9b479b, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/6f9c9ab5a6c849efaf93edfb28ccd599, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/d47d83e411454e1dafb3cba5dd35d173, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/0128002f234b476a96ab44c3673d9c40, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/9968c6ebf6f3465fbd2aa1386f0da5b3, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/1e689ab59b6940a1a3fe7a317284fa97, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/70979e5b912d4edca01e353d15bd864e, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/143f1559c61540bf935b24eba0dce8ed, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/97c7a80fdc1a43a3971af7fc8bcb1389, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/100acd5d34c04f5f8ed65c205de40ad6, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/f19a3f2872d44ca38d6848c07bbffa42, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/5bef005fccb24d4bbe133e6a055521b9, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/6a8729a388474378b0d3654abe32ab97, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/5eddcd3ef3c54ae585ab34880246d542, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/2f31b946fd3243e2b6f7990173c590e5, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/adada88cdd41440abcc8112c37adc0c7, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/437b3ced73424ae7999edca4385667e3, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/88fcd411f1ff4a5b950b32ba71239e91] to archive 2024-11-25T12:59:31,747 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T12:59:31,748 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/26df2c6ab7a741e2ad34b54a06238814 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/26df2c6ab7a741e2ad34b54a06238814 2024-11-25T12:59:31,749 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/075b4de41cba48a59dc9e7d580f7d67d to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/075b4de41cba48a59dc9e7d580f7d67d 2024-11-25T12:59:31,750 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/0ddd0ca5d2c040e68b48a0d397fb9cea to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/0ddd0ca5d2c040e68b48a0d397fb9cea 2024-11-25T12:59:31,750 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/2b66783f288e4f9098d11afc71c24698 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/2b66783f288e4f9098d11afc71c24698 2024-11-25T12:59:31,751 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/d810f965a4044ebfa0e724befc983c29 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/d810f965a4044ebfa0e724befc983c29 2024-11-25T12:59:31,752 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/dc5a19e2d9fa4ced956635a857081496 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/dc5a19e2d9fa4ced956635a857081496 2024-11-25T12:59:31,752 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/e9dfe42e7c76473282927aa94baffaee to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/e9dfe42e7c76473282927aa94baffaee 2024-11-25T12:59:31,753 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/da9cb6997f47465088b16f58ad38078a to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/da9cb6997f47465088b16f58ad38078a 2024-11-25T12:59:31,754 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/c722365822cc4185a1742d01698b2b1f to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/c722365822cc4185a1742d01698b2b1f 2024-11-25T12:59:31,754 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/20a29e5a1b954d66bfb20df6cc9b479b to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/20a29e5a1b954d66bfb20df6cc9b479b 2024-11-25T12:59:31,755 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/6f9c9ab5a6c849efaf93edfb28ccd599 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/6f9c9ab5a6c849efaf93edfb28ccd599 2024-11-25T12:59:31,756 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/d47d83e411454e1dafb3cba5dd35d173 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/d47d83e411454e1dafb3cba5dd35d173 2024-11-25T12:59:31,757 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/0128002f234b476a96ab44c3673d9c40 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/0128002f234b476a96ab44c3673d9c40 2024-11-25T12:59:31,757 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/9968c6ebf6f3465fbd2aa1386f0da5b3 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/9968c6ebf6f3465fbd2aa1386f0da5b3 2024-11-25T12:59:31,758 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/1e689ab59b6940a1a3fe7a317284fa97 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/1e689ab59b6940a1a3fe7a317284fa97 2024-11-25T12:59:31,759 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/70979e5b912d4edca01e353d15bd864e to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/70979e5b912d4edca01e353d15bd864e 2024-11-25T12:59:31,759 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/143f1559c61540bf935b24eba0dce8ed to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/143f1559c61540bf935b24eba0dce8ed 2024-11-25T12:59:31,760 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/97c7a80fdc1a43a3971af7fc8bcb1389 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/97c7a80fdc1a43a3971af7fc8bcb1389 2024-11-25T12:59:31,761 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/100acd5d34c04f5f8ed65c205de40ad6 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/100acd5d34c04f5f8ed65c205de40ad6 2024-11-25T12:59:31,761 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/f19a3f2872d44ca38d6848c07bbffa42 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/f19a3f2872d44ca38d6848c07bbffa42 2024-11-25T12:59:31,762 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/5bef005fccb24d4bbe133e6a055521b9 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/5bef005fccb24d4bbe133e6a055521b9 2024-11-25T12:59:31,763 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/6a8729a388474378b0d3654abe32ab97 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/6a8729a388474378b0d3654abe32ab97 2024-11-25T12:59:31,763 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/5eddcd3ef3c54ae585ab34880246d542 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/5eddcd3ef3c54ae585ab34880246d542 2024-11-25T12:59:31,764 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/2f31b946fd3243e2b6f7990173c590e5 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/2f31b946fd3243e2b6f7990173c590e5 2024-11-25T12:59:31,765 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/adada88cdd41440abcc8112c37adc0c7 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/adada88cdd41440abcc8112c37adc0c7 2024-11-25T12:59:31,765 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/437b3ced73424ae7999edca4385667e3 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/437b3ced73424ae7999edca4385667e3 2024-11-25T12:59:31,766 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/88fcd411f1ff4a5b950b32ba71239e91 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/88fcd411f1ff4a5b950b32ba71239e91 2024-11-25T12:59:32,027 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=402, memsize=4.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/b601ab3c545b4f5dafd6410936e19ae7 2024-11-25T12:59:32,032 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/B/5994f9bcbafd451c8f10543001340818 is 50, key is test_row_1/B:col10/1732539570971/Put/seqid=0 2024-11-25T12:59:32,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742259_1435 (size=9857) 2024-11-25T12:59:32,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-25T12:59:32,436 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/B/5994f9bcbafd451c8f10543001340818 2024-11-25T12:59:32,445 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/C/f16e13d3ae9f4779aeb69ff6fe611896 is 50, key is test_row_1/C:col10/1732539570971/Put/seqid=0 2024-11-25T12:59:32,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742260_1436 (size=9857) 2024-11-25T12:59:32,456 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/C/f16e13d3ae9f4779aeb69ff6fe611896 2024-11-25T12:59:32,460 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/A/b601ab3c545b4f5dafd6410936e19ae7 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/b601ab3c545b4f5dafd6410936e19ae7 2024-11-25T12:59:32,463 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/b601ab3c545b4f5dafd6410936e19ae7, entries=100, sequenceid=402, filesize=22.0 K 2024-11-25T12:59:32,464 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/B/5994f9bcbafd451c8f10543001340818 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/5994f9bcbafd451c8f10543001340818 2024-11-25T12:59:32,466 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/5994f9bcbafd451c8f10543001340818, entries=100, sequenceid=402, filesize=9.6 K 2024-11-25T12:59:32,467 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/.tmp/C/f16e13d3ae9f4779aeb69ff6fe611896 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/f16e13d3ae9f4779aeb69ff6fe611896 2024-11-25T12:59:32,469 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/f16e13d3ae9f4779aeb69ff6fe611896, entries=100, sequenceid=402, filesize=9.6 K 2024-11-25T12:59:32,470 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for de45cbd8ecc09a85a2c9aae45053021b in 1311ms, sequenceid=402, compaction requested=true 2024-11-25T12:59:32,474 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/recovered.edits/405.seqid, newMaxSeqId=405, maxSeqId=4 2024-11-25T12:59:32,474 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. 2024-11-25T12:59:32,474 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1676): Region close journal for de45cbd8ecc09a85a2c9aae45053021b: Waiting for close lock at 1732539571159Running coprocessor pre-close hooks at 1732539571159Disabling compacts and flushes for region at 1732539571159Disabling writes for close at 1732539571159Obtaining lock to block concurrent updates at 1732539571159Preparing flush snapshotting stores in de45cbd8ecc09a85a2c9aae45053021b at 1732539571159Finished memstore snapshotting TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=36720, getOffHeapSize=0, getCellsCount=300 at 1732539571159Flushing stores of TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b. at 1732539571160 (+1 ms)Flushing de45cbd8ecc09a85a2c9aae45053021b/A: creating writer at 1732539571160Flushing mob file de45cbd8ecc09a85a2c9aae45053021b/A: appending metadata at 1732539571167 (+7 ms)Flushing mob file de45cbd8ecc09a85a2c9aae45053021b/A: closing flushed file at 1732539571167Flushing de45cbd8ecc09a85a2c9aae45053021b/A: appending metadata at 1732539571615 (+448 ms)Flushing de45cbd8ecc09a85a2c9aae45053021b/A: closing flushed file at 1732539571615Flushing de45cbd8ecc09a85a2c9aae45053021b/B: creating writer at 1732539572030 (+415 ms)Flushing de45cbd8ecc09a85a2c9aae45053021b/B: appending metadata at 1732539572032 (+2 ms)Flushing de45cbd8ecc09a85a2c9aae45053021b/B: closing flushed file at 1732539572032Flushing de45cbd8ecc09a85a2c9aae45053021b/C: creating writer at 1732539572441 (+409 ms)Flushing de45cbd8ecc09a85a2c9aae45053021b/C: appending metadata at 1732539572445 (+4 ms)Flushing de45cbd8ecc09a85a2c9aae45053021b/C: closing flushed file at 1732539572445Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3b659dc3: reopening flushed file at 1732539572459 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@17f5fb39: reopening flushed file at 1732539572463 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@32224d92: reopening flushed file at 1732539572466 (+3 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for de45cbd8ecc09a85a2c9aae45053021b in 1311ms, sequenceid=402, compaction requested=true at 1732539572470 (+4 ms)Writing region close event to WAL at 1732539572471 (+1 ms)Running coprocessor post-close hooks at 1732539572474 (+3 ms)Closed at 1732539572474 2024-11-25T12:59:32,476 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] handler.UnassignRegionHandler(157): Closed de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:32,476 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=105 updating hbase:meta row=de45cbd8ecc09a85a2c9aae45053021b, regionState=CLOSED 2024-11-25T12:59:32,477 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=106, ppid=105, state=RUNNABLE, hasLock=false; CloseRegionProcedure de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 because future has completed 2024-11-25T12:59:32,479 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-25T12:59:32,479 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; CloseRegionProcedure de45cbd8ecc09a85a2c9aae45053021b, server=1ff8009e463e,35905,1732539449246 in 1.4710 sec 2024-11-25T12:59:32,481 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=105, resume processing ppid=104 2024-11-25T12:59:32,481 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, ppid=104, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=de45cbd8ecc09a85a2c9aae45053021b, UNASSIGN in 1.4750 sec 2024-11-25T12:59:32,482 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-25T12:59:32,482 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4780 sec 2024-11-25T12:59:32,483 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732539572483"}]},"ts":"1732539572483"} 2024-11-25T12:59:32,484 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-25T12:59:32,484 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-25T12:59:32,485 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4880 sec 2024-11-25T12:59:33,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-25T12:59:33,160 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:59:33,161 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-25T12:59:33,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T12:59:33,163 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=107, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T12:59:33,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-25T12:59:33,164 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=107, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T12:59:33,165 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:33,167 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A, FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B, FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C, FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/recovered.edits] 2024-11-25T12:59:33,169 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/124ce5fa34ab48588aba02a57b55379f to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/124ce5fa34ab48588aba02a57b55379f 2024-11-25T12:59:33,170 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/b601ab3c545b4f5dafd6410936e19ae7 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/b601ab3c545b4f5dafd6410936e19ae7 2024-11-25T12:59:33,171 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/d711805a420343039ba6ea497882d935 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/A/d711805a420343039ba6ea497882d935 2024-11-25T12:59:33,173 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/5994f9bcbafd451c8f10543001340818 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/5994f9bcbafd451c8f10543001340818 2024-11-25T12:59:33,174 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/85532865d2bd4b1ab25e3acb1ab7f37f to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/85532865d2bd4b1ab25e3acb1ab7f37f 2024-11-25T12:59:33,175 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/ed988dafef6f42dcbcb2465c2ae40594 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/B/ed988dafef6f42dcbcb2465c2ae40594 2024-11-25T12:59:33,177 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/a205b5e0b9bf4435abd7061e8f592b62 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/a205b5e0b9bf4435abd7061e8f592b62 2024-11-25T12:59:33,178 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/f16e13d3ae9f4779aeb69ff6fe611896 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/C/f16e13d3ae9f4779aeb69ff6fe611896 2024-11-25T12:59:33,181 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/recovered.edits/405.seqid to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b/recovered.edits/405.seqid 2024-11-25T12:59:33,183 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:33,187 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-25T12:59:33,188 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-25T12:59:33,189 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-25T12:59:33,204 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125014dfe5576754de8aba4bcbbfbc2be95_de45cbd8ecc09a85a2c9aae45053021b to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125014dfe5576754de8aba4bcbbfbc2be95_de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:33,206 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411250a6da13d459a41b4a23570b3037f6621_de45cbd8ecc09a85a2c9aae45053021b to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411250a6da13d459a41b4a23570b3037f6621_de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:33,208 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411250c3555227eae4d7ebb928f492d4aa90a_de45cbd8ecc09a85a2c9aae45053021b to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411250c3555227eae4d7ebb928f492d4aa90a_de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:33,210 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411250e0239cb476843a597cee69ef27d2276_de45cbd8ecc09a85a2c9aae45053021b to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411250e0239cb476843a597cee69ef27d2276_de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:33,211 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411250e2960f3fe414151a73874fbf98b5bbd_de45cbd8ecc09a85a2c9aae45053021b to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411250e2960f3fe414151a73874fbf98b5bbd_de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:33,213 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125141d2322abcb4cdcbd73937ad41ea8eb_de45cbd8ecc09a85a2c9aae45053021b to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125141d2322abcb4cdcbd73937ad41ea8eb_de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:33,215 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125162efc581e134a9ba60599eaad38801a_de45cbd8ecc09a85a2c9aae45053021b to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125162efc581e134a9ba60599eaad38801a_de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:33,218 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411251ce71830dbe541008fb0f15fbf42de62_de45cbd8ecc09a85a2c9aae45053021b to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411251ce71830dbe541008fb0f15fbf42de62_de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:33,220 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112535083c98c95b46d3909834fb6311fd6b_de45cbd8ecc09a85a2c9aae45053021b to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112535083c98c95b46d3909834fb6311fd6b_de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:33,221 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112539590864c27547b6a944dcdb540b26b5_de45cbd8ecc09a85a2c9aae45053021b to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112539590864c27547b6a944dcdb540b26b5_de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:33,225 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112539ec670bb3a84a18aaf0e26e1eb51abe_de45cbd8ecc09a85a2c9aae45053021b to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112539ec670bb3a84a18aaf0e26e1eb51abe_de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:33,228 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411253ca58c3dc7e842b0abb51bd9450558e0_de45cbd8ecc09a85a2c9aae45053021b to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411253ca58c3dc7e842b0abb51bd9450558e0_de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:33,232 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411256e009a06099f45bebf9cb68db6f514d4_de45cbd8ecc09a85a2c9aae45053021b to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411256e009a06099f45bebf9cb68db6f514d4_de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:33,234 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112586974719299c48bd94d8c16fdf840002_de45cbd8ecc09a85a2c9aae45053021b to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112586974719299c48bd94d8c16fdf840002_de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:33,235 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125acb8672d443246ac8c5d9ba389340022_de45cbd8ecc09a85a2c9aae45053021b to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125acb8672d443246ac8c5d9ba389340022_de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:33,236 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125c002da0ff5a24c43ad5b8b62f189f67f_de45cbd8ecc09a85a2c9aae45053021b to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125c002da0ff5a24c43ad5b8b62f189f67f_de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:33,237 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125c1f6bf90f45c4c08adda9bff93940572_de45cbd8ecc09a85a2c9aae45053021b to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125c1f6bf90f45c4c08adda9bff93940572_de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:33,238 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125c2e7f95f407e4bdbb64bb9b096b838b3_de45cbd8ecc09a85a2c9aae45053021b to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125c2e7f95f407e4bdbb64bb9b096b838b3_de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:33,238 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125da1bf51813294678a46282b3b181a118_de45cbd8ecc09a85a2c9aae45053021b to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125da1bf51813294678a46282b3b181a118_de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:33,239 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125dbae1b8b34d94255abb2f47d26f9a4f4_de45cbd8ecc09a85a2c9aae45053021b to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125dbae1b8b34d94255abb2f47d26f9a4f4_de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:33,240 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125dd54718c1cf84a418eb3c1b85d53ec53_de45cbd8ecc09a85a2c9aae45053021b to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125dd54718c1cf84a418eb3c1b85d53ec53_de45cbd8ecc09a85a2c9aae45053021b 2024-11-25T12:59:33,241 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-25T12:59:33,244 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=107, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T12:59:33,253 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-25T12:59:33,261 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-25T12:59:33,263 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=107, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T12:59:33,263 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-25T12:59:33,263 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732539573263"}]},"ts":"9223372036854775807"} 2024-11-25T12:59:33,270 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-25T12:59:33,270 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => de45cbd8ecc09a85a2c9aae45053021b, NAME => 'TestAcidGuarantees,,1732539546007.de45cbd8ecc09a85a2c9aae45053021b.', STARTKEY => '', ENDKEY => ''}] 2024-11-25T12:59:33,270 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-25T12:59:33,271 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732539573270"}]},"ts":"9223372036854775807"} 2024-11-25T12:59:33,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-25T12:59:33,274 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-25T12:59:33,280 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=107, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T12:59:33,282 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 119 msec 2024-11-25T12:59:33,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-25T12:59:33,480 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-25T12:59:33,480 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:59:33,488 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=242 (was 236) Potentially hanging thread: RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-348484797_22 at /127.0.0.1:42122 [Waiting for operation #344] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-10 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-348484797_22 at /127.0.0.1:42030 [Waiting for operation #416] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1510017845_22 at /127.0.0.1:42124 [Waiting for operation #359] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-11 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1510017845_22 at /127.0.0.1:42136 [Waiting for operation #351] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1031 (was 944) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=900 (was 1010) 2024-11-25T12:59:33,496 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=242, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=1031, ProcessCount=11, AvailableMemoryMB=899 2024-11-25T12:59:33,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-25T12:59:33,498 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T12:59:33,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] procedure2.ProcedureExecutor(1139): Stored pid=108, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-25T12:59:33,499 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=108, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T12:59:33,500 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:33,500 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 108 2024-11-25T12:59:33,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108 2024-11-25T12:59:33,500 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=108, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T12:59:33,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742261_1437 (size=959) 2024-11-25T12:59:33,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108 2024-11-25T12:59:33,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108 2024-11-25T12:59:33,913 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a 2024-11-25T12:59:33,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742262_1438 (size=53) 2024-11-25T12:59:34,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108 2024-11-25T12:59:34,341 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:59:34,341 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 6a5867ce2299ba674b3f042fd1fb41c6, disabling compactions & flushes 2024-11-25T12:59:34,342 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:34,342 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:34,342 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. after waiting 0 ms 2024-11-25T12:59:34,342 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:34,342 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:34,342 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6a5867ce2299ba674b3f042fd1fb41c6: Waiting for close lock at 1732539574341Disabling compacts and flushes for region at 1732539574341Disabling writes for close at 1732539574342 (+1 ms)Writing region close event to WAL at 1732539574342Closed at 1732539574342 2024-11-25T12:59:34,343 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=108, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T12:59:34,343 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732539574343"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732539574343"}]},"ts":"1732539574343"} 2024-11-25T12:59:34,345 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T12:59:34,346 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=108, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T12:59:34,346 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732539574346"}]},"ts":"1732539574346"} 2024-11-25T12:59:34,348 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-25T12:59:34,348 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6a5867ce2299ba674b3f042fd1fb41c6, ASSIGN}] 2024-11-25T12:59:34,350 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6a5867ce2299ba674b3f042fd1fb41c6, ASSIGN 2024-11-25T12:59:34,351 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6a5867ce2299ba674b3f042fd1fb41c6, ASSIGN; state=OFFLINE, location=1ff8009e463e,35905,1732539449246; forceNewPlan=false, retain=false 2024-11-25T12:59:34,502 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=109 updating hbase:meta row=6a5867ce2299ba674b3f042fd1fb41c6, regionState=OPENING, regionLocation=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:34,504 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6a5867ce2299ba674b3f042fd1fb41c6, ASSIGN because future has completed 2024-11-25T12:59:34,504 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246}] 2024-11-25T12:59:34,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108 2024-11-25T12:59:34,658 INFO [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:34,658 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(7752): Opening region: {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} 2024-11-25T12:59:34,659 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 6a5867ce2299ba674b3f042fd1fb41c6 2024-11-25T12:59:34,659 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:59:34,659 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(7794): checking encryption for 6a5867ce2299ba674b3f042fd1fb41c6 2024-11-25T12:59:34,659 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(7797): checking classloading for 6a5867ce2299ba674b3f042fd1fb41c6 2024-11-25T12:59:34,663 INFO [StoreOpener-6a5867ce2299ba674b3f042fd1fb41c6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 6a5867ce2299ba674b3f042fd1fb41c6 2024-11-25T12:59:34,665 INFO [StoreOpener-6a5867ce2299ba674b3f042fd1fb41c6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6a5867ce2299ba674b3f042fd1fb41c6 columnFamilyName A 2024-11-25T12:59:34,665 DEBUG [StoreOpener-6a5867ce2299ba674b3f042fd1fb41c6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:34,665 INFO [StoreOpener-6a5867ce2299ba674b3f042fd1fb41c6-1 {}] regionserver.HStore(327): Store=6a5867ce2299ba674b3f042fd1fb41c6/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:59:34,665 INFO [StoreOpener-6a5867ce2299ba674b3f042fd1fb41c6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 6a5867ce2299ba674b3f042fd1fb41c6 2024-11-25T12:59:34,667 INFO [StoreOpener-6a5867ce2299ba674b3f042fd1fb41c6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6a5867ce2299ba674b3f042fd1fb41c6 columnFamilyName B 2024-11-25T12:59:34,667 DEBUG [StoreOpener-6a5867ce2299ba674b3f042fd1fb41c6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:34,667 INFO [StoreOpener-6a5867ce2299ba674b3f042fd1fb41c6-1 {}] regionserver.HStore(327): Store=6a5867ce2299ba674b3f042fd1fb41c6/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:59:34,667 INFO [StoreOpener-6a5867ce2299ba674b3f042fd1fb41c6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 6a5867ce2299ba674b3f042fd1fb41c6 2024-11-25T12:59:34,668 INFO [StoreOpener-6a5867ce2299ba674b3f042fd1fb41c6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6a5867ce2299ba674b3f042fd1fb41c6 columnFamilyName C 2024-11-25T12:59:34,669 DEBUG [StoreOpener-6a5867ce2299ba674b3f042fd1fb41c6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:34,669 INFO [StoreOpener-6a5867ce2299ba674b3f042fd1fb41c6-1 {}] regionserver.HStore(327): Store=6a5867ce2299ba674b3f042fd1fb41c6/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:59:34,669 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1038): replaying wal for 6a5867ce2299ba674b3f042fd1fb41c6 2024-11-25T12:59:34,670 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6 2024-11-25T12:59:34,670 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6 2024-11-25T12:59:34,679 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1048): stopping wal replay for 6a5867ce2299ba674b3f042fd1fb41c6 2024-11-25T12:59:34,679 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1060): Cleaning up temporary data for 6a5867ce2299ba674b3f042fd1fb41c6 2024-11-25T12:59:34,680 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-25T12:59:34,684 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1093): writing seq id for 6a5867ce2299ba674b3f042fd1fb41c6 2024-11-25T12:59:34,686 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T12:59:34,687 INFO [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1114): Opened 6a5867ce2299ba674b3f042fd1fb41c6; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62384657, jitterRate=-0.07039617002010345}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-25T12:59:34,687 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6a5867ce2299ba674b3f042fd1fb41c6 2024-11-25T12:59:34,688 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1006): Region open journal for 6a5867ce2299ba674b3f042fd1fb41c6: Running coprocessor pre-open hook at 1732539574659Writing region info on filesystem at 1732539574659Initializing all the Stores at 1732539574663 (+4 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732539574663Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732539574663Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732539574663Cleaning up temporary data from old regions at 1732539574679 (+16 ms)Running coprocessor post-open hooks at 1732539574687 (+8 ms)Region opened successfully at 1732539574688 (+1 ms) 2024-11-25T12:59:34,689 INFO [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., pid=110, masterSystemTime=1732539574655 2024-11-25T12:59:34,692 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=109 updating hbase:meta row=6a5867ce2299ba674b3f042fd1fb41c6, regionState=OPEN, openSeqNum=2, regionLocation=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:34,694 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:34,694 INFO [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:34,694 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=110, ppid=109, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 because future has completed 2024-11-25T12:59:34,697 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-25T12:59:34,697 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; OpenRegionProcedure 6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 in 191 msec 2024-11-25T12:59:34,699 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=109, resume processing ppid=108 2024-11-25T12:59:34,699 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, ppid=108, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6a5867ce2299ba674b3f042fd1fb41c6, ASSIGN in 349 msec 2024-11-25T12:59:34,700 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=108, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T12:59:34,700 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732539574700"}]},"ts":"1732539574700"} 2024-11-25T12:59:34,702 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-25T12:59:34,703 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=108, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T12:59:34,706 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2050 sec 2024-11-25T12:59:35,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108 2024-11-25T12:59:35,650 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:59:35,651 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d6d1845, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:59:35,651 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ff8009e463e,46289,-1 for getting cluster id 2024-11-25T12:59:35,651 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:59:35,652 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb972fd5-cdcb-422b-acfe-41fc80d301d9' 2024-11-25T12:59:35,652 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:59:35,652 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb972fd5-cdcb-422b-acfe-41fc80d301d9" 2024-11-25T12:59:35,653 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1321d7b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:59:35,653 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ff8009e463e,46289,-1] 2024-11-25T12:59:35,653 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:59:35,653 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:59:35,654 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42772, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:59:35,654 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c7cfecc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:59:35,654 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:59:35,655 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,35905,1732539449246, seqNum=-1] 2024-11-25T12:59:35,655 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:59:35,656 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56056, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:59:35,657 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-25T12:59:35,657 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 1ff8009e463e,46289,1732539448369 2024-11-25T12:59:35,657 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@40bcf0f4 2024-11-25T12:59:35,657 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-25T12:59:35,658 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42786, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-25T12:59:35,659 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@205b33d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:59:35,659 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ff8009e463e,46289,-1 for getting cluster id 2024-11-25T12:59:35,659 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:59:35,660 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb972fd5-cdcb-422b-acfe-41fc80d301d9' 2024-11-25T12:59:35,660 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:59:35,660 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb972fd5-cdcb-422b-acfe-41fc80d301d9" 2024-11-25T12:59:35,660 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41ee413f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:59:35,660 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ff8009e463e,46289,-1] 2024-11-25T12:59:35,660 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:59:35,661 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:59:35,661 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42804, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:59:35,662 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10c40b48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:59:35,662 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e100fd5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:59:35,662 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ff8009e463e,46289,-1 for getting cluster id 2024-11-25T12:59:35,662 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:59:35,663 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb972fd5-cdcb-422b-acfe-41fc80d301d9' 2024-11-25T12:59:35,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:59:35,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb972fd5-cdcb-422b-acfe-41fc80d301d9" 2024-11-25T12:59:35,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@428946d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:59:35,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ff8009e463e,46289,-1] 2024-11-25T12:59:35,664 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:59:35,664 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:59:35,664 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42832, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:59:35,665 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cb5908c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:59:35,665 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58e40afb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:59:35,665 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ff8009e463e,46289,-1 for getting cluster id 2024-11-25T12:59:35,665 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:59:35,666 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb972fd5-cdcb-422b-acfe-41fc80d301d9' 2024-11-25T12:59:35,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:59:35,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb972fd5-cdcb-422b-acfe-41fc80d301d9" 2024-11-25T12:59:35,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29b6af78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:59:35,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ff8009e463e,46289,-1] 2024-11-25T12:59:35,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:59:35,667 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:59:35,667 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42854, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:59:35,668 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32fb5807, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:59:35,671 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b9f4da0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:59:35,671 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ff8009e463e,46289,-1 for getting cluster id 2024-11-25T12:59:35,671 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:59:35,672 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb972fd5-cdcb-422b-acfe-41fc80d301d9' 2024-11-25T12:59:35,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:59:35,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb972fd5-cdcb-422b-acfe-41fc80d301d9" 2024-11-25T12:59:35,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d4787aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:59:35,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ff8009e463e,46289,-1] 2024-11-25T12:59:35,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:59:35,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:59:35,673 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42872, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:59:35,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48b096a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:59:35,675 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@107467c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:59:35,675 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ff8009e463e,46289,-1 for getting cluster id 2024-11-25T12:59:35,675 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:59:35,676 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb972fd5-cdcb-422b-acfe-41fc80d301d9' 2024-11-25T12:59:35,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:59:35,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb972fd5-cdcb-422b-acfe-41fc80d301d9" 2024-11-25T12:59:35,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6957a445, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:59:35,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ff8009e463e,46289,-1] 2024-11-25T12:59:35,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:59:35,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:59:35,677 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42890, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:59:35,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1321ce46, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:59:35,683 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c83334c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:59:35,683 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ff8009e463e,46289,-1 for getting cluster id 2024-11-25T12:59:35,684 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:59:35,684 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb972fd5-cdcb-422b-acfe-41fc80d301d9' 2024-11-25T12:59:35,684 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:59:35,685 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb972fd5-cdcb-422b-acfe-41fc80d301d9" 2024-11-25T12:59:35,685 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@384825cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:59:35,685 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ff8009e463e,46289,-1] 2024-11-25T12:59:35,685 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:59:35,685 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:59:35,686 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42908, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:59:35,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a50cb1c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:59:35,691 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3532b5c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:59:35,691 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ff8009e463e,46289,-1 for getting cluster id 2024-11-25T12:59:35,692 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:59:35,693 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb972fd5-cdcb-422b-acfe-41fc80d301d9' 2024-11-25T12:59:35,693 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:59:35,693 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb972fd5-cdcb-422b-acfe-41fc80d301d9" 2024-11-25T12:59:35,693 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ee7fdb0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:59:35,693 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ff8009e463e,46289,-1] 2024-11-25T12:59:35,693 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:59:35,694 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:59:35,695 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42920, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:59:35,697 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e06e5d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:59:35,702 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3088fab0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:59:35,702 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ff8009e463e,46289,-1 for getting cluster id 2024-11-25T12:59:35,702 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:59:35,703 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb972fd5-cdcb-422b-acfe-41fc80d301d9' 2024-11-25T12:59:35,703 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:59:35,703 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb972fd5-cdcb-422b-acfe-41fc80d301d9" 2024-11-25T12:59:35,703 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3de36b48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:59:35,703 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ff8009e463e,46289,-1] 2024-11-25T12:59:35,703 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:59:35,704 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:59:35,704 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42944, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:59:35,705 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@221b9447, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:59:35,706 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@663bb5d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:59:35,706 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ff8009e463e,46289,-1 for getting cluster id 2024-11-25T12:59:35,706 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:59:35,707 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb972fd5-cdcb-422b-acfe-41fc80d301d9' 2024-11-25T12:59:35,707 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:59:35,707 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb972fd5-cdcb-422b-acfe-41fc80d301d9" 2024-11-25T12:59:35,707 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75b9ab0a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:59:35,707 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ff8009e463e,46289,-1] 2024-11-25T12:59:35,707 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:59:35,707 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:59:35,708 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42970, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:59:35,709 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c55218c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:59:35,709 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b31ef7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:59:35,709 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ff8009e463e,46289,-1 for getting cluster id 2024-11-25T12:59:35,709 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:59:35,710 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb972fd5-cdcb-422b-acfe-41fc80d301d9' 2024-11-25T12:59:35,710 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:59:35,710 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb972fd5-cdcb-422b-acfe-41fc80d301d9" 2024-11-25T12:59:35,710 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@714514c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:59:35,711 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ff8009e463e,46289,-1] 2024-11-25T12:59:35,711 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:59:35,711 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:59:35,712 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42990, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:59:35,712 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66dace5a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:59:35,713 DEBUG [Thread-1987 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:59:35,714 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:59:35,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-25T12:59:35,715 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,35905,1732539449246, seqNum=-1] 2024-11-25T12:59:35,716 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:59:35,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-25T12:59:35,717 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56060, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:59:35,718 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:59:35,718 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:59:35,718 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:59:35,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2] 2024-11-25T12:59:35,719 DEBUG [Thread-1974 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:59:35,720 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,35905,1732539449246, seqNum=-1] 2024-11-25T12:59:35,720 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:59:35,721 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56072, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:59:35,723 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2] 2024-11-25T12:59:35,724 DEBUG [Thread-1976 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:59:35,725 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,35905,1732539449246, seqNum=-1] 2024-11-25T12:59:35,725 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:59:35,726 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56082, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:59:35,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2] 2024-11-25T12:59:35,727 DEBUG [Thread-1989 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:59:35,728 DEBUG [Thread-1978 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:59:35,728 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,35905,1732539449246, seqNum=-1] 2024-11-25T12:59:35,729 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:59:35,729 DEBUG [Thread-1985 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:59:35,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,35905,1732539449246, seqNum=-1] 2024-11-25T12:59:35,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:59:35,730 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,35905,1732539449246, seqNum=-1] 2024-11-25T12:59:35,730 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:59:35,731 DEBUG [Thread-1980 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:59:35,732 DEBUG [Thread-1982 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:59:35,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,35905,1732539449246, seqNum=-1] 2024-11-25T12:59:35,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:59:35,739 DEBUG [Thread-1991 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:59:35,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,35905,1732539449246, seqNum=-1] 2024-11-25T12:59:35,740 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,35905,1732539449246, seqNum=-1] 2024-11-25T12:59:35,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:59:35,741 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:59:35,742 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56096, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:59:35,742 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56100, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:59:35,742 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56114, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:59:35,742 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56124, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:59:35,743 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56108, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:59:35,743 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56140, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:59:35,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2] 2024-11-25T12:59:35,745 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2] 2024-11-25T12:59:35,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2] 2024-11-25T12:59:35,746 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2] 2024-11-25T12:59:35,746 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2] 2024-11-25T12:59:35,748 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2] 2024-11-25T12:59:35,749 DEBUG [Thread-1993 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:59:35,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 6a5867ce2299ba674b3f042fd1fb41c6 2024-11-25T12:59:35,751 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:59:35,751 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6a5867ce2299ba674b3f042fd1fb41c6 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-25T12:59:35,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,35905,1732539449246, seqNum=-1] 2024-11-25T12:59:35,755 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:59:35,759 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56148, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:59:35,761 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2] 2024-11-25T12:59:35,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-25T12:59:35,822 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/bce81b1adf2f4b8a8c6f849811591ea8 is 50, key is test_row_0/A:col10/1732539575750/Put/seqid=0 2024-11-25T12:59:35,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742263_1439 (size=12001) 2024-11-25T12:59:35,848 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/bce81b1adf2f4b8a8c6f849811591ea8 2024-11-25T12:59:35,871 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-25T12:59:35,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:35,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. as already flushing 2024-11-25T12:59:35,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:35,872 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:35,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:35,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:35,876 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/a7c6c10ee9bc41e9bc0380388941995a is 50, key is test_row_0/B:col10/1732539575750/Put/seqid=0 2024-11-25T12:59:35,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:35,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56100 deadline: 1732539635874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:35,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:35,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:35,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:35,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:35,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56124 deadline: 1732539635884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:35,898 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:35,898 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:35,898 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:35,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:35,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56072 deadline: 1732539635885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:35,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:35,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56082 deadline: 1732539635886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:35,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:35,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56108 deadline: 1732539635886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:35,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:35,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:35,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:35,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:35,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:35,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:35,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:35,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:35,901 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:35,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742264_1440 (size=12001) 2024-11-25T12:59:35,905 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/a7c6c10ee9bc41e9bc0380388941995a 2024-11-25T12:59:35,954 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/faae89c614b2438cb4b1d26073397ae6 is 50, key is test_row_0/C:col10/1732539575750/Put/seqid=0 2024-11-25T12:59:35,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742265_1441 (size=12001) 2024-11-25T12:59:35,998 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/faae89c614b2438cb4b1d26073397ae6 2024-11-25T12:59:36,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:36,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56100 deadline: 1732539635994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:36,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:36,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:36,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:36,007 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/bce81b1adf2f4b8a8c6f849811591ea8 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/bce81b1adf2f4b8a8c6f849811591ea8 2024-11-25T12:59:36,017 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/bce81b1adf2f4b8a8c6f849811591ea8, entries=150, sequenceid=12, filesize=11.7 K 2024-11-25T12:59:36,018 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/a7c6c10ee9bc41e9bc0380388941995a as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/a7c6c10ee9bc41e9bc0380388941995a 2024-11-25T12:59:36,024 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/a7c6c10ee9bc41e9bc0380388941995a, entries=150, sequenceid=12, filesize=11.7 K 2024-11-25T12:59:36,024 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/faae89c614b2438cb4b1d26073397ae6 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/faae89c614b2438cb4b1d26073397ae6 2024-11-25T12:59:36,025 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-25T12:59:36,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:36,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. as already flushing 2024-11-25T12:59:36,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:36,025 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:36,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:36,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:36,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-25T12:59:36,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:36,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56072 deadline: 1732539636010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:36,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:36,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:36,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:36,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:36,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56082 deadline: 1732539636010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:36,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:36,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:36,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:36,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:36,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56108 deadline: 1732539636010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:36,031 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:36,031 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:36,031 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:36,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:36,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56124 deadline: 1732539636011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:36,034 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:36,034 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:36,034 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:36,042 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/faae89c614b2438cb4b1d26073397ae6, entries=150, sequenceid=12, filesize=11.7 K 2024-11-25T12:59:36,043 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 6a5867ce2299ba674b3f042fd1fb41c6 in 292ms, sequenceid=12, compaction requested=false 2024-11-25T12:59:36,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6a5867ce2299ba674b3f042fd1fb41c6: 2024-11-25T12:59:36,179 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-25T12:59:36,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:36,179 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing 6a5867ce2299ba674b3f042fd1fb41c6 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-25T12:59:36,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/4bcb91af5f244b5c9eefcc1e99dd978c is 50, key is test_row_0/A:col10/1732539575881/Put/seqid=0 2024-11-25T12:59:36,217 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. as already flushing 2024-11-25T12:59:36,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 6a5867ce2299ba674b3f042fd1fb41c6 2024-11-25T12:59:36,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742266_1442 (size=12001) 2024-11-25T12:59:36,255 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/4bcb91af5f244b5c9eefcc1e99dd978c 2024-11-25T12:59:36,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:36,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56100 deadline: 1732539636251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:36,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:36,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:36,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:36,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:36,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56082 deadline: 1732539636266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:36,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:36,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:36,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:36,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:36,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56124 deadline: 1732539636267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:36,290 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:36,290 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:36,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:36,291 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:36,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56072 deadline: 1732539636268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:36,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:36,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:36,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:36,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:36,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56108 deadline: 1732539636273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:36,292 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:36,292 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:36,292 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:36,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/36376b0bc10c42d780ec9daa3f2bc865 is 50, key is test_row_0/B:col10/1732539575881/Put/seqid=0 2024-11-25T12:59:36,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742267_1443 (size=12001) 2024-11-25T12:59:36,333 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/36376b0bc10c42d780ec9daa3f2bc865 2024-11-25T12:59:36,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-25T12:59:36,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/0be2bedc113d4fb5a9866cab074458df is 50, key is test_row_0/C:col10/1732539575881/Put/seqid=0 2024-11-25T12:59:36,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:36,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56100 deadline: 1732539636381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:36,389 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:36,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:36,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:36,401 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:36,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56124 deadline: 1732539636401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:36,402 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:36,402 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:36,402 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:36,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:36,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56082 deadline: 1732539636401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:36,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:36,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:36,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:36,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:36,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56108 deadline: 1732539636401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:36,409 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:36,409 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:36,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:36,409 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:36,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56072 deadline: 1732539636401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:36,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:36,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:36,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:36,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742268_1444 (size=12001) 2024-11-25T12:59:36,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:36,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56100 deadline: 1732539636600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:36,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:36,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:36,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:36,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:36,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56124 deadline: 1732539636610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:36,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:36,616 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:36,616 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:36,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56082 deadline: 1732539636610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:36,616 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:36,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:36,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:36,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:36,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:36,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56108 deadline: 1732539636620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:36,626 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:36,626 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:36,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:36,627 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:36,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56072 deadline: 1732539636621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:36,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:36,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:36,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:36,821 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/0be2bedc113d4fb5a9866cab074458df 2024-11-25T12:59:36,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/4bcb91af5f244b5c9eefcc1e99dd978c as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/4bcb91af5f244b5c9eefcc1e99dd978c 2024-11-25T12:59:36,832 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/4bcb91af5f244b5c9eefcc1e99dd978c, entries=150, sequenceid=37, filesize=11.7 K 2024-11-25T12:59:36,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/36376b0bc10c42d780ec9daa3f2bc865 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/36376b0bc10c42d780ec9daa3f2bc865 2024-11-25T12:59:36,839 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/36376b0bc10c42d780ec9daa3f2bc865, entries=150, sequenceid=37, filesize=11.7 K 2024-11-25T12:59:36,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/0be2bedc113d4fb5a9866cab074458df as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/0be2bedc113d4fb5a9866cab074458df 2024-11-25T12:59:36,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-25T12:59:36,859 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/0be2bedc113d4fb5a9866cab074458df, entries=150, sequenceid=37, filesize=11.7 K 2024-11-25T12:59:36,862 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 6a5867ce2299ba674b3f042fd1fb41c6 in 683ms, sequenceid=37, compaction requested=false 2024-11-25T12:59:36,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for 6a5867ce2299ba674b3f042fd1fb41c6: 2024-11-25T12:59:36,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:36,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-25T12:59:36,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-25T12:59:36,872 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-25T12:59:36,872 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1480 sec 2024-11-25T12:59:36,879 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 1.1590 sec 2024-11-25T12:59:36,912 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:59:36,912 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6a5867ce2299ba674b3f042fd1fb41c6 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-25T12:59:36,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 6a5867ce2299ba674b3f042fd1fb41c6 2024-11-25T12:59:36,921 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/90fcc36a71984e4597da967c8d21e35e is 50, key is test_row_0/A:col10/1732539576257/Put/seqid=0 2024-11-25T12:59:36,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742269_1445 (size=14341) 2024-11-25T12:59:36,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:36,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56100 deadline: 1732539636976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:36,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:36,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:36,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:36,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:36,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56082 deadline: 1732539636977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:36,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:36,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56072 deadline: 1732539636978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:36,993 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:36,993 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:36,993 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:36,995 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:36,995 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:36,995 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:36,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:36,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56108 deadline: 1732539636986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:36,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:36,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56124 deadline: 1732539636987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:37,000 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:37,001 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:37,001 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:37,001 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:37,001 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:37,001 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:37,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:37,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56100 deadline: 1732539637103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:37,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:37,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:37,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56072 deadline: 1732539637104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:37,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:37,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:37,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56082 deadline: 1732539637105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:37,110 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:37,110 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:37,110 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:37,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:37,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:37,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:37,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:37,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:37,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56108 deadline: 1732539637111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:37,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:37,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56124 deadline: 1732539637111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:37,122 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:37,122 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:37,122 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:37,122 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:37,122 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:37,122 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:37,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:37,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56072 deadline: 1732539637320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:37,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:37,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:37,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:37,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:37,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56100 deadline: 1732539637320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:37,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:37,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:37,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:37,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:37,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56082 deadline: 1732539637320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:37,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:37,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:37,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:37,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:37,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56108 deadline: 1732539637330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:37,341 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:37,341 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:37,341 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:37,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:37,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56124 deadline: 1732539637330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:37,342 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:37,342 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:37,342 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:37,347 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/90fcc36a71984e4597da967c8d21e35e 2024-11-25T12:59:37,393 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/9ea8e2cd914a4ed0b5582d5802fcc928 is 50, key is test_row_0/B:col10/1732539576257/Put/seqid=0 2024-11-25T12:59:37,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742270_1446 (size=12001) 2024-11-25T12:59:37,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:37,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56082 deadline: 1732539637632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:37,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:37,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:37,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:37,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:37,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56100 deadline: 1732539637640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:37,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:37,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:37,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:37,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:37,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56072 deadline: 1732539637644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:37,665 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:37,665 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:37,665 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:37,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:37,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56124 deadline: 1732539637655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:37,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:37,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56108 deadline: 1732539637657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:37,674 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:37,674 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:37,674 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:37,674 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:37,674 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:37,674 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:37,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-25T12:59:37,860 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:59:37,860 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:59:37,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-25T12:59:37,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-25T12:59:37,867 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:59:37,868 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:59:37,868 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:59:37,874 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/9ea8e2cd914a4ed0b5582d5802fcc928 2024-11-25T12:59:37,904 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/6eb0620f29654f479eb5b07e620013f1 is 50, key is test_row_0/C:col10/1732539576257/Put/seqid=0 2024-11-25T12:59:37,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742271_1447 (size=12001) 2024-11-25T12:59:37,944 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/6eb0620f29654f479eb5b07e620013f1 2024-11-25T12:59:37,948 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/90fcc36a71984e4597da967c8d21e35e as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/90fcc36a71984e4597da967c8d21e35e 2024-11-25T12:59:37,955 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/90fcc36a71984e4597da967c8d21e35e, entries=200, sequenceid=49, filesize=14.0 K 2024-11-25T12:59:37,956 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/9ea8e2cd914a4ed0b5582d5802fcc928 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/9ea8e2cd914a4ed0b5582d5802fcc928 2024-11-25T12:59:37,962 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/9ea8e2cd914a4ed0b5582d5802fcc928, entries=150, sequenceid=49, filesize=11.7 K 2024-11-25T12:59:37,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/6eb0620f29654f479eb5b07e620013f1 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/6eb0620f29654f479eb5b07e620013f1 2024-11-25T12:59:37,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-25T12:59:37,985 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/6eb0620f29654f479eb5b07e620013f1, entries=150, sequenceid=49, filesize=11.7 K 2024-11-25T12:59:37,986 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 6a5867ce2299ba674b3f042fd1fb41c6 in 1074ms, sequenceid=49, compaction requested=true 2024-11-25T12:59:37,986 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6a5867ce2299ba674b3f042fd1fb41c6: 2024-11-25T12:59:37,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6a5867ce2299ba674b3f042fd1fb41c6:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:59:37,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:59:37,986 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:59:37,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6a5867ce2299ba674b3f042fd1fb41c6:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:59:37,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:59:37,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6a5867ce2299ba674b3f042fd1fb41c6:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:59:37,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-25T12:59:37,987 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:59:37,987 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:59:37,987 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:59:37,988 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): 6a5867ce2299ba674b3f042fd1fb41c6/B is initiating minor compaction (all files) 2024-11-25T12:59:37,988 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1541): 6a5867ce2299ba674b3f042fd1fb41c6/A is initiating minor compaction (all files) 2024-11-25T12:59:37,988 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6a5867ce2299ba674b3f042fd1fb41c6/B in TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:37,988 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6a5867ce2299ba674b3f042fd1fb41c6/A in TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:37,988 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/bce81b1adf2f4b8a8c6f849811591ea8, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/4bcb91af5f244b5c9eefcc1e99dd978c, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/90fcc36a71984e4597da967c8d21e35e] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp, totalSize=37.4 K 2024-11-25T12:59:37,988 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/a7c6c10ee9bc41e9bc0380388941995a, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/36376b0bc10c42d780ec9daa3f2bc865, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/9ea8e2cd914a4ed0b5582d5802fcc928] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp, totalSize=35.2 K 2024-11-25T12:59:37,988 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting a7c6c10ee9bc41e9bc0380388941995a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732539575747 2024-11-25T12:59:37,988 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting bce81b1adf2f4b8a8c6f849811591ea8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732539575747 2024-11-25T12:59:37,988 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4bcb91af5f244b5c9eefcc1e99dd978c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732539575872 2024-11-25T12:59:37,988 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 36376b0bc10c42d780ec9daa3f2bc865, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732539575872 2024-11-25T12:59:37,988 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90fcc36a71984e4597da967c8d21e35e, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732539576249 2024-11-25T12:59:37,988 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ea8e2cd914a4ed0b5582d5802fcc928, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732539576257 2024-11-25T12:59:38,014 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6a5867ce2299ba674b3f042fd1fb41c6#A#compaction#368 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:59:38,014 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/97e8728d9ac142aa8727fe3ff69e4e27 is 50, key is test_row_0/A:col10/1732539576257/Put/seqid=0 2024-11-25T12:59:38,020 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-25T12:59:38,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:38,020 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing 6a5867ce2299ba674b3f042fd1fb41c6 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-25T12:59:38,026 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T12:59:38,031 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6a5867ce2299ba674b3f042fd1fb41c6#B#compaction#369 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:59:38,032 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/9ddd08755bd54490aa83720c6aecfb85 is 50, key is test_row_0/B:col10/1732539576257/Put/seqid=0 2024-11-25T12:59:38,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/6c724c6f1bad493289b80feaa8273e21 is 50, key is test_row_0/A:col10/1732539576983/Put/seqid=0 2024-11-25T12:59:38,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742272_1448 (size=12104) 2024-11-25T12:59:38,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742273_1449 (size=12104) 2024-11-25T12:59:38,134 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/97e8728d9ac142aa8727fe3ff69e4e27 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/97e8728d9ac142aa8727fe3ff69e4e27 2024-11-25T12:59:38,149 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6a5867ce2299ba674b3f042fd1fb41c6/A of 6a5867ce2299ba674b3f042fd1fb41c6 into 97e8728d9ac142aa8727fe3ff69e4e27(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:59:38,149 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6a5867ce2299ba674b3f042fd1fb41c6: 2024-11-25T12:59:38,149 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., storeName=6a5867ce2299ba674b3f042fd1fb41c6/A, priority=13, startTime=1732539577986; duration=0sec 2024-11-25T12:59:38,149 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:59:38,149 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6a5867ce2299ba674b3f042fd1fb41c6:A 2024-11-25T12:59:38,149 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:59:38,151 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:59:38,151 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1541): 6a5867ce2299ba674b3f042fd1fb41c6/C is initiating minor compaction (all files) 2024-11-25T12:59:38,151 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6a5867ce2299ba674b3f042fd1fb41c6/C in TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:38,151 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/faae89c614b2438cb4b1d26073397ae6, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/0be2bedc113d4fb5a9866cab074458df, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/6eb0620f29654f479eb5b07e620013f1] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp, totalSize=35.2 K 2024-11-25T12:59:38,151 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting faae89c614b2438cb4b1d26073397ae6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732539575747 2024-11-25T12:59:38,152 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0be2bedc113d4fb5a9866cab074458df, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732539575872 2024-11-25T12:59:38,152 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6eb0620f29654f479eb5b07e620013f1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732539576257 2024-11-25T12:59:38,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742274_1450 (size=12001) 2024-11-25T12:59:38,170 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/9ddd08755bd54490aa83720c6aecfb85 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/9ddd08755bd54490aa83720c6aecfb85 2024-11-25T12:59:38,173 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6a5867ce2299ba674b3f042fd1fb41c6#C#compaction#371 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:59:38,174 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/965fa3de64834ea089094935e82579d5 is 50, key is test_row_0/C:col10/1732539576257/Put/seqid=0 2024-11-25T12:59:38,180 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6a5867ce2299ba674b3f042fd1fb41c6/B of 6a5867ce2299ba674b3f042fd1fb41c6 into 9ddd08755bd54490aa83720c6aecfb85(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:59:38,180 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6a5867ce2299ba674b3f042fd1fb41c6: 2024-11-25T12:59:38,180 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., storeName=6a5867ce2299ba674b3f042fd1fb41c6/B, priority=13, startTime=1732539577986; duration=0sec 2024-11-25T12:59:38,180 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:59:38,180 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6a5867ce2299ba674b3f042fd1fb41c6:B 2024-11-25T12:59:38,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-25T12:59:38,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. as already flushing 2024-11-25T12:59:38,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 6a5867ce2299ba674b3f042fd1fb41c6 2024-11-25T12:59:38,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742275_1451 (size=12104) 2024-11-25T12:59:38,229 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/965fa3de64834ea089094935e82579d5 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/965fa3de64834ea089094935e82579d5 2024-11-25T12:59:38,242 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6a5867ce2299ba674b3f042fd1fb41c6/C of 6a5867ce2299ba674b3f042fd1fb41c6 into 965fa3de64834ea089094935e82579d5(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:59:38,242 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6a5867ce2299ba674b3f042fd1fb41c6: 2024-11-25T12:59:38,242 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., storeName=6a5867ce2299ba674b3f042fd1fb41c6/C, priority=13, startTime=1732539577986; duration=0sec 2024-11-25T12:59:38,242 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:59:38,242 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6a5867ce2299ba674b3f042fd1fb41c6:C 2024-11-25T12:59:38,245 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:38,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56100 deadline: 1732539638228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:38,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:38,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:38,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:38,249 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:38,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56072 deadline: 1732539638229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:38,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:38,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:38,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:38,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56124 deadline: 1732539638233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:38,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:38,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:38,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56108 deadline: 1732539638233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:38,251 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:38,251 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:38,251 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:38,252 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:38,252 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:38,252 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:38,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:38,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56082 deadline: 1732539638241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:38,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:38,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:38,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:38,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:38,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56100 deadline: 1732539638352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:38,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:38,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:38,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:38,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:38,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56082 deadline: 1732539638360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:38,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:38,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:38,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:38,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:38,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56108 deadline: 1732539638360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:38,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:38,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56072 deadline: 1732539638360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:38,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:38,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56124 deadline: 1732539638361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:38,369 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:38,369 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:38,369 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:38,369 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:38,369 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:38,369 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:38,369 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:38,369 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:38,369 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:38,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-25T12:59:38,559 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/6c724c6f1bad493289b80feaa8273e21 2024-11-25T12:59:38,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:38,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56082 deadline: 1732539638570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:38,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:38,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:38,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:38,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:38,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56100 deadline: 1732539638570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:38,579 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:38,579 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:38,579 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:38,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/acaf5b1589d8495696880c05a36e4860 is 50, key is test_row_0/B:col10/1732539576983/Put/seqid=0 2024-11-25T12:59:38,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:38,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56124 deadline: 1732539638580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:38,590 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:38,590 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:38,590 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:38,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:38,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56108 deadline: 1732539638580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:38,591 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:38,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:38,591 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:38,591 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:38,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56072 deadline: 1732539638580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:38,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:38,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:38,592 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:38,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742276_1452 (size=12001) 2024-11-25T12:59:38,640 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/acaf5b1589d8495696880c05a36e4860 2024-11-25T12:59:38,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/8a91b22160264d5bb578957e259377a3 is 50, key is test_row_0/C:col10/1732539576983/Put/seqid=0 2024-11-25T12:59:38,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742277_1453 (size=12001) 2024-11-25T12:59:38,731 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/8a91b22160264d5bb578957e259377a3 2024-11-25T12:59:38,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/6c724c6f1bad493289b80feaa8273e21 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/6c724c6f1bad493289b80feaa8273e21 2024-11-25T12:59:38,751 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/6c724c6f1bad493289b80feaa8273e21, entries=150, sequenceid=73, filesize=11.7 K 2024-11-25T12:59:38,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/acaf5b1589d8495696880c05a36e4860 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/acaf5b1589d8495696880c05a36e4860 2024-11-25T12:59:38,756 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/acaf5b1589d8495696880c05a36e4860, entries=150, sequenceid=73, filesize=11.7 K 2024-11-25T12:59:38,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/8a91b22160264d5bb578957e259377a3 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/8a91b22160264d5bb578957e259377a3 2024-11-25T12:59:38,761 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/8a91b22160264d5bb578957e259377a3, entries=150, sequenceid=73, filesize=11.7 K 2024-11-25T12:59:38,762 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=73.80 KB/75570 for 6a5867ce2299ba674b3f042fd1fb41c6 in 742ms, sequenceid=73, compaction requested=false 2024-11-25T12:59:38,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for 6a5867ce2299ba674b3f042fd1fb41c6: 2024-11-25T12:59:38,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:38,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-25T12:59:38,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-25T12:59:38,765 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-25T12:59:38,765 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 896 msec 2024-11-25T12:59:38,766 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 905 msec 2024-11-25T12:59:38,903 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:59:38,903 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6a5867ce2299ba674b3f042fd1fb41c6 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-25T12:59:38,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 6a5867ce2299ba674b3f042fd1fb41c6 2024-11-25T12:59:38,935 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/6b19b9811a2a46f8abb7344a29b3336e is 50, key is test_row_0/A:col10/1732539578901/Put/seqid=0 2024-11-25T12:59:38,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742278_1454 (size=14341) 2024-11-25T12:59:38,962 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/6b19b9811a2a46f8abb7344a29b3336e 2024-11-25T12:59:38,985 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/69b147cb40c14cf899e8348053abaf9a is 50, key is test_row_0/B:col10/1732539578901/Put/seqid=0 2024-11-25T12:59:38,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742279_1455 (size=12001) 2024-11-25T12:59:39,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-25T12:59:39,000 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:59:39,000 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:59:39,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-25T12:59:39,002 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:59:39,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-25T12:59:39,004 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:59:39,004 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:59:39,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:39,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56124 deadline: 1732539638992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:39,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:39,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56072 deadline: 1732539638993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:39,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:39,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56100 deadline: 1732539638994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:39,007 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:39,007 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:39,007 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:39,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:39,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:39,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:39,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:39,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:39,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:39,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:39,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56082 deadline: 1732539639005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:39,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:39,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:39,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:39,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:39,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56108 deadline: 1732539639006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:39,015 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:39,015 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:39,015 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:39,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-25T12:59:39,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:39,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56124 deadline: 1732539639110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:39,118 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:39,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:39,118 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:39,118 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:39,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56100 deadline: 1732539639110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:39,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:39,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:39,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:39,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:39,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56072 deadline: 1732539639110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:39,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:39,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:39,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:39,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:39,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56082 deadline: 1732539639120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:39,128 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:39,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:39,128 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:39,128 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:39,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56108 deadline: 1732539639120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:39,128 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:39,128 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:39,128 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:39,156 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-25T12:59:39,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:39,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. as already flushing 2024-11-25T12:59:39,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:39,156 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:39,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:39,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:39,309 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-25T12:59:39,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:39,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. as already flushing 2024-11-25T12:59:39,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:39,309 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:39,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:39,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:39,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-25T12:59:39,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:39,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56124 deadline: 1732539639330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:39,334 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:39,334 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:39,334 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:39,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:39,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56100 deadline: 1732539639330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:39,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:39,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:39,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:39,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56072 deadline: 1732539639330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:39,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:39,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:39,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:39,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:39,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:39,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56108 deadline: 1732539639347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:39,348 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:39,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:39,348 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:39,348 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:39,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56082 deadline: 1732539639347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:39,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:39,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:39,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:39,390 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/69b147cb40c14cf899e8348053abaf9a 2024-11-25T12:59:39,409 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/5186f903f012462788987471f8f8c839 is 50, key is test_row_0/C:col10/1732539578901/Put/seqid=0 2024-11-25T12:59:39,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742280_1456 (size=12001) 2024-11-25T12:59:39,448 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/5186f903f012462788987471f8f8c839 2024-11-25T12:59:39,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/6b19b9811a2a46f8abb7344a29b3336e as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/6b19b9811a2a46f8abb7344a29b3336e 2024-11-25T12:59:39,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-25T12:59:39,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:39,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. as already flushing 2024-11-25T12:59:39,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:39,462 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:39,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:39,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:39,476 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/6b19b9811a2a46f8abb7344a29b3336e, entries=200, sequenceid=91, filesize=14.0 K 2024-11-25T12:59:39,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/69b147cb40c14cf899e8348053abaf9a as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/69b147cb40c14cf899e8348053abaf9a 2024-11-25T12:59:39,481 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/69b147cb40c14cf899e8348053abaf9a, entries=150, sequenceid=91, filesize=11.7 K 2024-11-25T12:59:39,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/5186f903f012462788987471f8f8c839 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/5186f903f012462788987471f8f8c839 2024-11-25T12:59:39,487 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/5186f903f012462788987471f8f8c839, entries=150, sequenceid=91, filesize=11.7 K 2024-11-25T12:59:39,494 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 6a5867ce2299ba674b3f042fd1fb41c6 in 591ms, sequenceid=91, compaction requested=true 2024-11-25T12:59:39,494 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6a5867ce2299ba674b3f042fd1fb41c6: 2024-11-25T12:59:39,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6a5867ce2299ba674b3f042fd1fb41c6:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:59:39,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:59:39,494 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:59:39,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6a5867ce2299ba674b3f042fd1fb41c6:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:59:39,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:59:39,494 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:59:39,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6a5867ce2299ba674b3f042fd1fb41c6:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:59:39,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:59:39,499 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:59:39,499 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1541): 6a5867ce2299ba674b3f042fd1fb41c6/A is initiating minor compaction (all files) 2024-11-25T12:59:39,499 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6a5867ce2299ba674b3f042fd1fb41c6/A in TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:39,500 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/97e8728d9ac142aa8727fe3ff69e4e27, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/6c724c6f1bad493289b80feaa8273e21, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/6b19b9811a2a46f8abb7344a29b3336e] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp, totalSize=37.5 K 2024-11-25T12:59:39,500 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97e8728d9ac142aa8727fe3ff69e4e27, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732539576257 2024-11-25T12:59:39,500 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:59:39,500 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): 6a5867ce2299ba674b3f042fd1fb41c6/B is initiating minor compaction (all files) 2024-11-25T12:59:39,500 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6a5867ce2299ba674b3f042fd1fb41c6/B in TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:39,500 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/9ddd08755bd54490aa83720c6aecfb85, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/acaf5b1589d8495696880c05a36e4860, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/69b147cb40c14cf899e8348053abaf9a] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp, totalSize=35.3 K 2024-11-25T12:59:39,501 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ddd08755bd54490aa83720c6aecfb85, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732539576257 2024-11-25T12:59:39,501 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c724c6f1bad493289b80feaa8273e21, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1732539576972 2024-11-25T12:59:39,501 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting acaf5b1589d8495696880c05a36e4860, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1732539576972 2024-11-25T12:59:39,501 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b19b9811a2a46f8abb7344a29b3336e, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732539578200 2024-11-25T12:59:39,501 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 69b147cb40c14cf899e8348053abaf9a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732539578200 2024-11-25T12:59:39,522 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6a5867ce2299ba674b3f042fd1fb41c6#B#compaction#377 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:59:39,523 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/01780359b71342cc81992d65a78b358d is 50, key is test_row_0/B:col10/1732539578901/Put/seqid=0 2024-11-25T12:59:39,532 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6a5867ce2299ba674b3f042fd1fb41c6#A#compaction#378 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:59:39,533 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/cf48b19c7a8a4d7ebda03d3f11056d47 is 50, key is test_row_0/A:col10/1732539578901/Put/seqid=0 2024-11-25T12:59:39,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742282_1458 (size=12207) 2024-11-25T12:59:39,613 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/01780359b71342cc81992d65a78b358d as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/01780359b71342cc81992d65a78b358d 2024-11-25T12:59:39,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742281_1457 (size=12207) 2024-11-25T12:59:39,620 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6a5867ce2299ba674b3f042fd1fb41c6/B of 6a5867ce2299ba674b3f042fd1fb41c6 into 01780359b71342cc81992d65a78b358d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:59:39,620 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6a5867ce2299ba674b3f042fd1fb41c6: 2024-11-25T12:59:39,620 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., storeName=6a5867ce2299ba674b3f042fd1fb41c6/B, priority=13, startTime=1732539579494; duration=0sec 2024-11-25T12:59:39,620 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:59:39,620 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6a5867ce2299ba674b3f042fd1fb41c6:B 2024-11-25T12:59:39,621 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:59:39,621 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:59:39,621 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): 6a5867ce2299ba674b3f042fd1fb41c6/C is initiating minor compaction (all files) 2024-11-25T12:59:39,621 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6a5867ce2299ba674b3f042fd1fb41c6/C in TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:39,621 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/965fa3de64834ea089094935e82579d5, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/8a91b22160264d5bb578957e259377a3, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/5186f903f012462788987471f8f8c839] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp, totalSize=35.3 K 2024-11-25T12:59:39,622 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 965fa3de64834ea089094935e82579d5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732539576257 2024-11-25T12:59:39,622 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a91b22160264d5bb578957e259377a3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1732539576972 2024-11-25T12:59:39,622 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 5186f903f012462788987471f8f8c839, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732539578200 2024-11-25T12:59:39,626 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/cf48b19c7a8a4d7ebda03d3f11056d47 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/cf48b19c7a8a4d7ebda03d3f11056d47 2024-11-25T12:59:39,626 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-25T12:59:39,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:39,627 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing 6a5867ce2299ba674b3f042fd1fb41c6 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-25T12:59:39,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-25T12:59:39,641 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6a5867ce2299ba674b3f042fd1fb41c6/A of 6a5867ce2299ba674b3f042fd1fb41c6 into cf48b19c7a8a4d7ebda03d3f11056d47(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:59:39,641 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6a5867ce2299ba674b3f042fd1fb41c6: 2024-11-25T12:59:39,641 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., storeName=6a5867ce2299ba674b3f042fd1fb41c6/A, priority=13, startTime=1732539579494; duration=0sec 2024-11-25T12:59:39,642 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:59:39,642 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6a5867ce2299ba674b3f042fd1fb41c6:A 2024-11-25T12:59:39,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/7cabbe60ed29439291d864fd27ec933e is 50, key is test_row_0/A:col10/1732539579004/Put/seqid=0 2024-11-25T12:59:39,651 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. as already flushing 2024-11-25T12:59:39,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 6a5867ce2299ba674b3f042fd1fb41c6 2024-11-25T12:59:39,658 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6a5867ce2299ba674b3f042fd1fb41c6#C#compaction#380 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:59:39,658 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/8255c7cce7ce4b87abea7212fa804499 is 50, key is test_row_0/C:col10/1732539578901/Put/seqid=0 2024-11-25T12:59:39,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742284_1460 (size=12207) 2024-11-25T12:59:39,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:39,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56072 deadline: 1732539639672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:39,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:39,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:39,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:39,680 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/8255c7cce7ce4b87abea7212fa804499 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/8255c7cce7ce4b87abea7212fa804499 2024-11-25T12:59:39,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:39,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56082 deadline: 1732539639674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:39,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:39,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:39,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:39,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:39,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56124 deadline: 1732539639675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:39,685 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:39,685 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:39,685 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:39,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:39,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56100 deadline: 1732539639676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:39,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:39,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:39,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:39,688 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6a5867ce2299ba674b3f042fd1fb41c6/C of 6a5867ce2299ba674b3f042fd1fb41c6 into 8255c7cce7ce4b87abea7212fa804499(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:59:39,688 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6a5867ce2299ba674b3f042fd1fb41c6: 2024-11-25T12:59:39,688 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., storeName=6a5867ce2299ba674b3f042fd1fb41c6/C, priority=13, startTime=1732539579494; duration=0sec 2024-11-25T12:59:39,688 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:59:39,688 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6a5867ce2299ba674b3f042fd1fb41c6:C 2024-11-25T12:59:39,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:39,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56108 deadline: 1732539639677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:39,689 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:39,690 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:39,690 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:39,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742283_1459 (size=12001) 2024-11-25T12:59:39,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:39,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56082 deadline: 1732539639790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:39,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:39,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56072 deadline: 1732539639790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:39,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:39,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:39,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:39,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:39,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56100 deadline: 1732539639790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:39,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:39,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:39,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:39,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:39,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56124 deadline: 1732539639791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:39,799 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:39,799 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:39,799 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:39,803 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:39,803 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:39,803 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:39,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:39,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56108 deadline: 1732539639803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:39,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:39,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:39,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:40,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:40,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56072 deadline: 1732539640000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:40,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:40,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:40,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:40,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:40,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56124 deadline: 1732539640010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:40,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:40,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56082 deadline: 1732539640010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:40,020 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:40,020 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:40,020 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:40,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:40,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56100 deadline: 1732539640011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:40,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:40,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:40,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:40,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:40,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:40,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:40,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:40,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56108 deadline: 1732539640020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:40,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:40,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:40,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:40,099 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/7cabbe60ed29439291d864fd27ec933e 2024-11-25T12:59:40,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/8904b306030d42ce903606ba9f8be7b9 is 50, key is test_row_0/B:col10/1732539579004/Put/seqid=0 2024-11-25T12:59:40,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-25T12:59:40,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742285_1461 (size=12001) 2024-11-25T12:59:40,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:40,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56072 deadline: 1732539640320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:40,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:40,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:40,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:40,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:40,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56124 deadline: 1732539640330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:40,335 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:40,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:40,335 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:40,335 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:40,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56082 deadline: 1732539640330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:40,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:40,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:40,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:40,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:40,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56100 deadline: 1732539640330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:40,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:40,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:40,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:40,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:40,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56108 deadline: 1732539640330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:40,336 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:40,336 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:40,336 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:40,562 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/8904b306030d42ce903606ba9f8be7b9 2024-11-25T12:59:40,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/bcc6ad99feb24b56a76a919d0c414713 is 50, key is test_row_0/C:col10/1732539579004/Put/seqid=0 2024-11-25T12:59:40,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742286_1462 (size=12001) 2024-11-25T12:59:40,577 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/bcc6ad99feb24b56a76a919d0c414713 2024-11-25T12:59:40,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/7cabbe60ed29439291d864fd27ec933e as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/7cabbe60ed29439291d864fd27ec933e 2024-11-25T12:59:40,586 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/7cabbe60ed29439291d864fd27ec933e, entries=150, sequenceid=113, filesize=11.7 K 2024-11-25T12:59:40,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/8904b306030d42ce903606ba9f8be7b9 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/8904b306030d42ce903606ba9f8be7b9 2024-11-25T12:59:40,592 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/8904b306030d42ce903606ba9f8be7b9, entries=150, sequenceid=113, filesize=11.7 K 2024-11-25T12:59:40,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/bcc6ad99feb24b56a76a919d0c414713 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/bcc6ad99feb24b56a76a919d0c414713 2024-11-25T12:59:40,596 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/bcc6ad99feb24b56a76a919d0c414713, entries=150, sequenceid=113, filesize=11.7 K 2024-11-25T12:59:40,599 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 6a5867ce2299ba674b3f042fd1fb41c6 in 973ms, sequenceid=113, compaction requested=false 2024-11-25T12:59:40,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for 6a5867ce2299ba674b3f042fd1fb41c6: 2024-11-25T12:59:40,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:40,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-25T12:59:40,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-25T12:59:40,602 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-25T12:59:40,602 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5960 sec 2024-11-25T12:59:40,607 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 1.6030 sec 2024-11-25T12:59:40,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 6a5867ce2299ba674b3f042fd1fb41c6 2024-11-25T12:59:40,858 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:59:40,858 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6a5867ce2299ba674b3f042fd1fb41c6 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-25T12:59:40,879 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/35ba11c833b64979810e0ae19386de1a is 50, key is test_row_0/A:col10/1732539580855/Put/seqid=0 2024-11-25T12:59:40,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742287_1463 (size=21565) 2024-11-25T12:59:40,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:40,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56100 deadline: 1732539640940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:40,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:40,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:40,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:40,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56124 deadline: 1732539640942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:40,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:40,960 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:40,960 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:40,960 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:40,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:40,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56108 deadline: 1732539640944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:40,963 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:40,963 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:40,963 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:40,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:40,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56072 deadline: 1732539640945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:40,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:40,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:40,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:40,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:40,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56082 deadline: 1732539640946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:40,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:40,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:40,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:41,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:41,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56100 deadline: 1732539641076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:41,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:41,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:41,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:41,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:41,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56124 deadline: 1732539641076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:41,082 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:41,082 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:41,082 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:41,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:41,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56108 deadline: 1732539641076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:41,082 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:41,082 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:41,082 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:41,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:41,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56082 deadline: 1732539641076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:41,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:41,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:41,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:41,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:41,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56072 deadline: 1732539641076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:41,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:41,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:41,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:41,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-25T12:59:41,150 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:59:41,151 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:59:41,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-25T12:59:41,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-25T12:59:41,156 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:59:41,156 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:59:41,156 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:59:41,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-25T12:59:41,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:41,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56100 deadline: 1732539641290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:41,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:41,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:41,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:41,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:41,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56108 deadline: 1732539641290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:41,300 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:41,300 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:41,300 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:41,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:41,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56082 deadline: 1732539641290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:41,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:41,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:41,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:41,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:41,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56072 deadline: 1732539641290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:41,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:41,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:41,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:41,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:41,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56124 deadline: 1732539641290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:41,302 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:41,302 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:41,302 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:41,309 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-25T12:59:41,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:41,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. as already flushing 2024-11-25T12:59:41,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:41,309 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:41,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:41,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:41,320 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/35ba11c833b64979810e0ae19386de1a 2024-11-25T12:59:41,357 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/d0e431729c2f47daaf371948849e2f93 is 50, key is test_row_0/B:col10/1732539580855/Put/seqid=0 2024-11-25T12:59:41,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742288_1464 (size=12151) 2024-11-25T12:59:41,397 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/d0e431729c2f47daaf371948849e2f93 2024-11-25T12:59:41,420 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/05c97d1de0f44baa83dd80d54ae9dcc5 is 50, key is test_row_0/C:col10/1732539580855/Put/seqid=0 2024-11-25T12:59:41,463 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-25T12:59:41,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:41,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. as already flushing 2024-11-25T12:59:41,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:41,464 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:41,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742289_1465 (size=12151) 2024-11-25T12:59:41,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:41,465 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/05c97d1de0f44baa83dd80d54ae9dcc5 2024-11-25T12:59:41,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:41,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-25T12:59:41,472 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/35ba11c833b64979810e0ae19386de1a as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/35ba11c833b64979810e0ae19386de1a 2024-11-25T12:59:41,487 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/35ba11c833b64979810e0ae19386de1a, entries=350, sequenceid=134, filesize=21.1 K 2024-11-25T12:59:41,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/d0e431729c2f47daaf371948849e2f93 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/d0e431729c2f47daaf371948849e2f93 2024-11-25T12:59:41,504 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/d0e431729c2f47daaf371948849e2f93, entries=150, sequenceid=134, filesize=11.9 K 2024-11-25T12:59:41,510 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/05c97d1de0f44baa83dd80d54ae9dcc5 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/05c97d1de0f44baa83dd80d54ae9dcc5 2024-11-25T12:59:41,530 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/05c97d1de0f44baa83dd80d54ae9dcc5, entries=150, sequenceid=134, filesize=11.9 K 2024-11-25T12:59:41,534 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 6a5867ce2299ba674b3f042fd1fb41c6 in 676ms, sequenceid=134, compaction requested=true 2024-11-25T12:59:41,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6a5867ce2299ba674b3f042fd1fb41c6: 2024-11-25T12:59:41,534 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:59:41,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6a5867ce2299ba674b3f042fd1fb41c6:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:59:41,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:59:41,534 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:59:41,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6a5867ce2299ba674b3f042fd1fb41c6:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:59:41,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:59:41,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6a5867ce2299ba674b3f042fd1fb41c6:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:59:41,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:59:41,536 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:59:41,536 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): 6a5867ce2299ba674b3f042fd1fb41c6/B is initiating minor compaction (all files) 2024-11-25T12:59:41,536 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6a5867ce2299ba674b3f042fd1fb41c6/B in TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:41,536 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/01780359b71342cc81992d65a78b358d, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/8904b306030d42ce903606ba9f8be7b9, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/d0e431729c2f47daaf371948849e2f93] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp, totalSize=35.5 K 2024-11-25T12:59:41,537 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 01780359b71342cc81992d65a78b358d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732539578200 2024-11-25T12:59:41,537 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 8904b306030d42ce903606ba9f8be7b9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1732539578992 2024-11-25T12:59:41,537 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 45773 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:59:41,537 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1541): 6a5867ce2299ba674b3f042fd1fb41c6/A is initiating minor compaction (all files) 2024-11-25T12:59:41,537 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6a5867ce2299ba674b3f042fd1fb41c6/A in TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:41,537 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/cf48b19c7a8a4d7ebda03d3f11056d47, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/7cabbe60ed29439291d864fd27ec933e, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/35ba11c833b64979810e0ae19386de1a] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp, totalSize=44.7 K 2024-11-25T12:59:41,538 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf48b19c7a8a4d7ebda03d3f11056d47, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732539578200 2024-11-25T12:59:41,538 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting d0e431729c2f47daaf371948849e2f93, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732539580854 2024-11-25T12:59:41,538 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7cabbe60ed29439291d864fd27ec933e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1732539578992 2024-11-25T12:59:41,539 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35ba11c833b64979810e0ae19386de1a, keycount=350, bloomtype=ROW, size=21.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732539579672 2024-11-25T12:59:41,569 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6a5867ce2299ba674b3f042fd1fb41c6#B#compaction#386 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:59:41,570 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/38820115db0840338ee44e56420aafd7 is 50, key is test_row_0/B:col10/1732539580855/Put/seqid=0 2024-11-25T12:59:41,590 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6a5867ce2299ba674b3f042fd1fb41c6#A#compaction#387 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:59:41,590 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/72441409117d4bbfa349b45ccb6a07a3 is 50, key is test_row_0/A:col10/1732539580855/Put/seqid=0 2024-11-25T12:59:41,620 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:59:41,621 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6a5867ce2299ba674b3f042fd1fb41c6 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-25T12:59:41,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 6a5867ce2299ba674b3f042fd1fb41c6 2024-11-25T12:59:41,637 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-25T12:59:41,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:41,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. as already flushing 2024-11-25T12:59:41,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:41,638 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:41,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:41,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:41,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742290_1466 (size=12459) 2024-11-25T12:59:41,653 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/38820115db0840338ee44e56420aafd7 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/38820115db0840338ee44e56420aafd7 2024-11-25T12:59:41,657 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6a5867ce2299ba674b3f042fd1fb41c6/B of 6a5867ce2299ba674b3f042fd1fb41c6 into 38820115db0840338ee44e56420aafd7(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:59:41,657 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6a5867ce2299ba674b3f042fd1fb41c6: 2024-11-25T12:59:41,657 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., storeName=6a5867ce2299ba674b3f042fd1fb41c6/B, priority=13, startTime=1732539581534; duration=0sec 2024-11-25T12:59:41,657 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:59:41,657 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6a5867ce2299ba674b3f042fd1fb41c6:B 2024-11-25T12:59:41,657 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:59:41,658 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:59:41,658 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): 6a5867ce2299ba674b3f042fd1fb41c6/C is initiating minor compaction (all files) 2024-11-25T12:59:41,658 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6a5867ce2299ba674b3f042fd1fb41c6/C in TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:41,658 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/8255c7cce7ce4b87abea7212fa804499, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/bcc6ad99feb24b56a76a919d0c414713, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/05c97d1de0f44baa83dd80d54ae9dcc5] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp, totalSize=35.5 K 2024-11-25T12:59:41,659 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 8255c7cce7ce4b87abea7212fa804499, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732539578200 2024-11-25T12:59:41,659 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting bcc6ad99feb24b56a76a919d0c414713, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1732539578992 2024-11-25T12:59:41,659 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 05c97d1de0f44baa83dd80d54ae9dcc5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732539580854 2024-11-25T12:59:41,662 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/a5b2a413b848452d98a761dc104f29ed is 50, key is test_row_0/A:col10/1732539580943/Put/seqid=0 2024-11-25T12:59:41,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742291_1467 (size=12459) 2024-11-25T12:59:41,685 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6a5867ce2299ba674b3f042fd1fb41c6#C#compaction#389 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:59:41,686 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/26382852102148f4a936c279df67a892 is 50, key is test_row_0/C:col10/1732539580855/Put/seqid=0 2024-11-25T12:59:41,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:41,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56108 deadline: 1732539641671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:41,687 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:41,687 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:41,687 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:41,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:41,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56082 deadline: 1732539641673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:41,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:41,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:41,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:41,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:41,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56072 deadline: 1732539641674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:41,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:41,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:41,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:41,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:41,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56124 deadline: 1732539641675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:41,694 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:41,694 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:41,694 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:41,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:41,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56100 deadline: 1732539641676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:41,694 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:41,694 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:41,694 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:41,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742292_1468 (size=14541) 2024-11-25T12:59:41,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742293_1469 (size=12459) 2024-11-25T12:59:41,737 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/a5b2a413b848452d98a761dc104f29ed 2024-11-25T12:59:41,749 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/1bd17b8424c14cdd91cbe8baf004ce2b is 50, key is test_row_0/B:col10/1732539580943/Put/seqid=0 2024-11-25T12:59:41,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-25T12:59:41,796 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-25T12:59:41,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:41,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. as already flushing 2024-11-25T12:59:41,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:41,796 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:41,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:41,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:41,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742294_1470 (size=12151) 2024-11-25T12:59:41,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:41,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56108 deadline: 1732539641791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:41,805 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:41,805 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:41,805 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:41,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:41,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56082 deadline: 1732539641802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:41,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:41,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:41,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:41,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:41,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56072 deadline: 1732539641802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:41,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:41,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:41,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:41,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:41,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56124 deadline: 1732539641802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:41,817 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:41,817 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:41,817 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:41,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:41,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56100 deadline: 1732539641803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:41,818 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:41,818 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:41,818 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:41,952 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-25T12:59:41,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:41,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. as already flushing 2024-11-25T12:59:41,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:41,952 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:41,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:41,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:42,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:42,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56108 deadline: 1732539642010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:42,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:42,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:42,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:42,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:42,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56082 deadline: 1732539642020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:42,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:42,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:42,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:42,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:42,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56072 deadline: 1732539642020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:42,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:42,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:42,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:42,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56100 deadline: 1732539642020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:42,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:42,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:42,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:42,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:42,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:42,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56124 deadline: 1732539642030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:42,045 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:42,045 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:42,045 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:42,068 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/72441409117d4bbfa349b45ccb6a07a3 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/72441409117d4bbfa349b45ccb6a07a3 2024-11-25T12:59:42,073 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6a5867ce2299ba674b3f042fd1fb41c6/A of 6a5867ce2299ba674b3f042fd1fb41c6 into 72441409117d4bbfa349b45ccb6a07a3(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:59:42,073 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6a5867ce2299ba674b3f042fd1fb41c6: 2024-11-25T12:59:42,073 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., storeName=6a5867ce2299ba674b3f042fd1fb41c6/A, priority=13, startTime=1732539581534; duration=0sec 2024-11-25T12:59:42,074 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:59:42,074 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6a5867ce2299ba674b3f042fd1fb41c6:A 2024-11-25T12:59:42,109 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-25T12:59:42,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:42,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. as already flushing 2024-11-25T12:59:42,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:42,111 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:42,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:42,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:42,142 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/26382852102148f4a936c279df67a892 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/26382852102148f4a936c279df67a892 2024-11-25T12:59:42,149 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6a5867ce2299ba674b3f042fd1fb41c6/C of 6a5867ce2299ba674b3f042fd1fb41c6 into 26382852102148f4a936c279df67a892(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:59:42,149 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6a5867ce2299ba674b3f042fd1fb41c6: 2024-11-25T12:59:42,149 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., storeName=6a5867ce2299ba674b3f042fd1fb41c6/C, priority=13, startTime=1732539581535; duration=0sec 2024-11-25T12:59:42,149 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:59:42,149 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6a5867ce2299ba674b3f042fd1fb41c6:C 2024-11-25T12:59:42,201 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/1bd17b8424c14cdd91cbe8baf004ce2b 2024-11-25T12:59:42,229 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/15447463641a45b491208e08f896cae6 is 50, key is test_row_0/C:col10/1732539580943/Put/seqid=0 2024-11-25T12:59:42,268 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-25T12:59:42,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:42,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. as already flushing 2024-11-25T12:59:42,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:42,268 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:42,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:42,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:42,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742295_1471 (size=12151) 2024-11-25T12:59:42,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-25T12:59:42,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:42,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56108 deadline: 1732539642335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:42,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:42,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:42,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:42,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:42,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56082 deadline: 1732539642341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:42,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:42,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:42,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:42,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:42,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56072 deadline: 1732539642341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:42,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:42,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:42,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:42,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:42,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56100 deadline: 1732539642343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:42,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:42,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:42,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:42,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:42,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56124 deadline: 1732539642351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:42,362 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:42,362 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:42,363 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:42,422 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-25T12:59:42,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:42,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. as already flushing 2024-11-25T12:59:42,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:42,423 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:42,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:42,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:42,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-25T12:59:42,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:42,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. as already flushing 2024-11-25T12:59:42,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:42,580 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:42,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:42,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:42,692 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/15447463641a45b491208e08f896cae6 2024-11-25T12:59:42,697 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/a5b2a413b848452d98a761dc104f29ed as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/a5b2a413b848452d98a761dc104f29ed 2024-11-25T12:59:42,702 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/a5b2a413b848452d98a761dc104f29ed, entries=200, sequenceid=152, filesize=14.2 K 2024-11-25T12:59:42,706 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/1bd17b8424c14cdd91cbe8baf004ce2b as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/1bd17b8424c14cdd91cbe8baf004ce2b 2024-11-25T12:59:42,713 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/1bd17b8424c14cdd91cbe8baf004ce2b, entries=150, sequenceid=152, filesize=11.9 K 2024-11-25T12:59:42,714 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/15447463641a45b491208e08f896cae6 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/15447463641a45b491208e08f896cae6 2024-11-25T12:59:42,720 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/15447463641a45b491208e08f896cae6, entries=150, sequenceid=152, filesize=11.9 K 2024-11-25T12:59:42,721 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 6a5867ce2299ba674b3f042fd1fb41c6 in 1101ms, sequenceid=152, compaction requested=false 2024-11-25T12:59:42,721 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6a5867ce2299ba674b3f042fd1fb41c6: 2024-11-25T12:59:42,744 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-25T12:59:42,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:42,744 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 6a5867ce2299ba674b3f042fd1fb41c6 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-25T12:59:42,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/d11f5e665b914943b0acffa4f1be33fd is 50, key is test_row_0/A:col10/1732539581651/Put/seqid=0 2024-11-25T12:59:42,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742296_1472 (size=12151) 2024-11-25T12:59:42,833 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/d11f5e665b914943b0acffa4f1be33fd 2024-11-25T12:59:42,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/c7c485d518714f4cad1de6e4a6ec35f4 is 50, key is test_row_0/B:col10/1732539581651/Put/seqid=0 2024-11-25T12:59:42,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 6a5867ce2299ba674b3f042fd1fb41c6 2024-11-25T12:59:42,880 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. as already flushing 2024-11-25T12:59:42,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742297_1473 (size=12151) 2024-11-25T12:59:42,908 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/c7c485d518714f4cad1de6e4a6ec35f4 2024-11-25T12:59:42,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/483313348eab4a8e9ff723d02a9dcead is 50, key is test_row_0/C:col10/1732539581651/Put/seqid=0 2024-11-25T12:59:42,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:42,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56072 deadline: 1732539642927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:42,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:42,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56100 deadline: 1732539642925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:42,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:42,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:42,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:42,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:42,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:42,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:42,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:42,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56082 deadline: 1732539642937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:42,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:42,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:42,966 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:42,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:42,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56108 deadline: 1732539642938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:42,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:42,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56124 deadline: 1732539642942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:42,968 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:42,968 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:42,968 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:42,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:42,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:42,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:42,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742298_1474 (size=12151) 2024-11-25T12:59:43,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:43,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56072 deadline: 1732539643070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:43,079 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:43,079 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:43,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:43,079 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:43,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56100 deadline: 1732539643070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:43,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:43,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:43,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:43,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:43,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56082 deadline: 1732539643070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:43,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:43,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:43,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:43,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:43,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56124 deadline: 1732539643071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:43,081 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:43,081 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:43,081 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:43,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:43,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56108 deadline: 1732539643080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:43,088 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:43,088 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:43,088 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:43,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:43,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56100 deadline: 1732539643291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:43,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:43,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:43,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:43,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56124 deadline: 1732539643291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:43,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:43,303 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:43,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:43,303 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:43,303 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:43,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56082 deadline: 1732539643291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:43,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:43,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:43,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:43,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:43,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56072 deadline: 1732539643292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:43,304 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:43,304 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:43,304 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:43,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:43,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56108 deadline: 1732539643300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:43,308 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:43,308 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:43,308 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:43,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-25T12:59:43,396 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/483313348eab4a8e9ff723d02a9dcead 2024-11-25T12:59:43,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/d11f5e665b914943b0acffa4f1be33fd as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/d11f5e665b914943b0acffa4f1be33fd 2024-11-25T12:59:43,415 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/d11f5e665b914943b0acffa4f1be33fd, entries=150, sequenceid=173, filesize=11.9 K 2024-11-25T12:59:43,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/c7c485d518714f4cad1de6e4a6ec35f4 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/c7c485d518714f4cad1de6e4a6ec35f4 2024-11-25T12:59:43,424 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/c7c485d518714f4cad1de6e4a6ec35f4, entries=150, sequenceid=173, filesize=11.9 K 2024-11-25T12:59:43,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/483313348eab4a8e9ff723d02a9dcead as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/483313348eab4a8e9ff723d02a9dcead 2024-11-25T12:59:43,429 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/483313348eab4a8e9ff723d02a9dcead, entries=150, sequenceid=173, filesize=11.9 K 2024-11-25T12:59:43,429 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 6a5867ce2299ba674b3f042fd1fb41c6 in 685ms, sequenceid=173, compaction requested=true 2024-11-25T12:59:43,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 6a5867ce2299ba674b3f042fd1fb41c6: 2024-11-25T12:59:43,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:43,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-25T12:59:43,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-25T12:59:43,438 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-25T12:59:43,438 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2810 sec 2024-11-25T12:59:43,439 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 2.2870 sec 2024-11-25T12:59:43,621 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:59:43,621 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6a5867ce2299ba674b3f042fd1fb41c6 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-25T12:59:43,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 6a5867ce2299ba674b3f042fd1fb41c6 2024-11-25T12:59:43,637 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/1788c4e191bb45bfbb999d0358791ab1 is 50, key is test_row_0/A:col10/1732539583614/Put/seqid=0 2024-11-25T12:59:43,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:43,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56100 deadline: 1732539643657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:43,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:43,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:43,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:43,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:43,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56082 deadline: 1732539643659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:43,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:43,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:43,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:43,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:43,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56124 deadline: 1732539643660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:43,686 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:43,686 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:43,686 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:43,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742299_1475 (size=12151) 2024-11-25T12:59:43,692 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/1788c4e191bb45bfbb999d0358791ab1 2024-11-25T12:59:43,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:43,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56108 deadline: 1732539643679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:43,712 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:43,713 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:43,713 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:43,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:43,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56072 deadline: 1732539643684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:43,713 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:43,713 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:43,713 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:43,741 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/98acac1c2461476d8ae643667c3dfc8d is 50, key is test_row_0/B:col10/1732539583614/Put/seqid=0 2024-11-25T12:59:43,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742300_1476 (size=12151) 2024-11-25T12:59:43,792 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/98acac1c2461476d8ae643667c3dfc8d 2024-11-25T12:59:43,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:43,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56100 deadline: 1732539643792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:43,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:43,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:43,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:43,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:43,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56124 deadline: 1732539643792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:43,809 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:43,809 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:43,809 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:43,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:43,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56082 deadline: 1732539643792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:43,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:43,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:43,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:43,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:43,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56108 deadline: 1732539643820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:43,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:43,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56072 deadline: 1732539643821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:43,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:43,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:43,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:43,838 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:43,838 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:43,838 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:43,840 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/b09eceb035c1467b8870b9d89f8b0929 is 50, key is test_row_0/C:col10/1732539583614/Put/seqid=0 2024-11-25T12:59:43,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742301_1477 (size=12151) 2024-11-25T12:59:44,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:44,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56100 deadline: 1732539644015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:44,028 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:44,028 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:44,028 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:44,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:44,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56082 deadline: 1732539644028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:44,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:44,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:44,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:44,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:44,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56124 deadline: 1732539644029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:44,044 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:44,044 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:44,044 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:44,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:44,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56072 deadline: 1732539644043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:44,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:44,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:44,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:44,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:44,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56108 deadline: 1732539644050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:44,063 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:44,063 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:44,063 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:44,289 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/b09eceb035c1467b8870b9d89f8b0929 2024-11-25T12:59:44,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/1788c4e191bb45bfbb999d0358791ab1 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/1788c4e191bb45bfbb999d0358791ab1 2024-11-25T12:59:44,300 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/1788c4e191bb45bfbb999d0358791ab1, entries=150, sequenceid=193, filesize=11.9 K 2024-11-25T12:59:44,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/98acac1c2461476d8ae643667c3dfc8d as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/98acac1c2461476d8ae643667c3dfc8d 2024-11-25T12:59:44,309 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/98acac1c2461476d8ae643667c3dfc8d, entries=150, sequenceid=193, filesize=11.9 K 2024-11-25T12:59:44,310 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/b09eceb035c1467b8870b9d89f8b0929 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/b09eceb035c1467b8870b9d89f8b0929 2024-11-25T12:59:44,324 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/b09eceb035c1467b8870b9d89f8b0929, entries=150, sequenceid=193, filesize=11.9 K 2024-11-25T12:59:44,325 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 6a5867ce2299ba674b3f042fd1fb41c6 in 704ms, sequenceid=193, compaction requested=true 2024-11-25T12:59:44,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6a5867ce2299ba674b3f042fd1fb41c6: 2024-11-25T12:59:44,326 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:59:44,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6a5867ce2299ba674b3f042fd1fb41c6:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:59:44,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:59:44,327 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:59:44,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6a5867ce2299ba674b3f042fd1fb41c6:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:59:44,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:59:44,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6a5867ce2299ba674b3f042fd1fb41c6:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:59:44,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:59:44,339 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51302 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:59:44,340 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1541): 6a5867ce2299ba674b3f042fd1fb41c6/A is initiating minor compaction (all files) 2024-11-25T12:59:44,340 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6a5867ce2299ba674b3f042fd1fb41c6/A in TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:44,340 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/72441409117d4bbfa349b45ccb6a07a3, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/a5b2a413b848452d98a761dc104f29ed, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/d11f5e665b914943b0acffa4f1be33fd, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/1788c4e191bb45bfbb999d0358791ab1] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp, totalSize=50.1 K 2024-11-25T12:59:44,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 6a5867ce2299ba674b3f042fd1fb41c6 2024-11-25T12:59:44,343 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:59:44,343 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): 6a5867ce2299ba674b3f042fd1fb41c6/B is initiating minor compaction (all files) 2024-11-25T12:59:44,343 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72441409117d4bbfa349b45ccb6a07a3, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732539580854 2024-11-25T12:59:44,344 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6a5867ce2299ba674b3f042fd1fb41c6/B in TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:44,344 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/38820115db0840338ee44e56420aafd7, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/1bd17b8424c14cdd91cbe8baf004ce2b, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/c7c485d518714f4cad1de6e4a6ec35f4, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/98acac1c2461476d8ae643667c3dfc8d] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp, totalSize=47.8 K 2024-11-25T12:59:44,344 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:59:44,344 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6a5867ce2299ba674b3f042fd1fb41c6 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-25T12:59:44,344 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5b2a413b848452d98a761dc104f29ed, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1732539580940 2024-11-25T12:59:44,344 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 38820115db0840338ee44e56420aafd7, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732539580854 2024-11-25T12:59:44,345 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 1bd17b8424c14cdd91cbe8baf004ce2b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1732539580943 2024-11-25T12:59:44,345 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting d11f5e665b914943b0acffa4f1be33fd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732539581651 2024-11-25T12:59:44,345 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting c7c485d518714f4cad1de6e4a6ec35f4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732539581651 2024-11-25T12:59:44,345 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1788c4e191bb45bfbb999d0358791ab1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732539582926 2024-11-25T12:59:44,345 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 98acac1c2461476d8ae643667c3dfc8d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732539582926 2024-11-25T12:59:44,348 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/3983032290e34afc8aeb0bb748ebe92a is 50, key is test_row_0/A:col10/1732539583638/Put/seqid=0 2024-11-25T12:59:44,390 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6a5867ce2299ba674b3f042fd1fb41c6#A#compaction#399 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:59:44,390 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/8f2c51c193dc4b0b87f1f5cf2c9d28a4 is 50, key is test_row_0/A:col10/1732539583614/Put/seqid=0 2024-11-25T12:59:44,410 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6a5867ce2299ba674b3f042fd1fb41c6#B#compaction#400 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:59:44,410 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/87c623cd79a342f7bd6acb4f73d6c004 is 50, key is test_row_0/B:col10/1732539583614/Put/seqid=0 2024-11-25T12:59:44,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742302_1478 (size=14541) 2024-11-25T12:59:44,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:44,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56072 deadline: 1732539644428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:44,455 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:44,455 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:44,455 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:44,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742304_1480 (size=12595) 2024-11-25T12:59:44,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:44,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56124 deadline: 1732539644448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:44,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:44,483 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:44,483 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:44,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56100 deadline: 1732539644451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:44,483 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:44,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:44,483 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:44,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:44,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:44,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56108 deadline: 1732539644453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:44,487 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:44,487 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:44,487 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:44,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742303_1479 (size=12595) 2024-11-25T12:59:44,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:44,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56082 deadline: 1732539644464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:44,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:44,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:44,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:44,504 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/8f2c51c193dc4b0b87f1f5cf2c9d28a4 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/8f2c51c193dc4b0b87f1f5cf2c9d28a4 2024-11-25T12:59:44,512 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6a5867ce2299ba674b3f042fd1fb41c6/A of 6a5867ce2299ba674b3f042fd1fb41c6 into 8f2c51c193dc4b0b87f1f5cf2c9d28a4(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:59:44,512 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6a5867ce2299ba674b3f042fd1fb41c6: 2024-11-25T12:59:44,512 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., storeName=6a5867ce2299ba674b3f042fd1fb41c6/A, priority=12, startTime=1732539584326; duration=0sec 2024-11-25T12:59:44,512 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:59:44,512 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6a5867ce2299ba674b3f042fd1fb41c6:A 2024-11-25T12:59:44,512 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:59:44,513 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:59:44,513 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1541): 6a5867ce2299ba674b3f042fd1fb41c6/C is initiating minor compaction (all files) 2024-11-25T12:59:44,513 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6a5867ce2299ba674b3f042fd1fb41c6/C in TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:44,514 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/26382852102148f4a936c279df67a892, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/15447463641a45b491208e08f896cae6, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/483313348eab4a8e9ff723d02a9dcead, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/b09eceb035c1467b8870b9d89f8b0929] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp, totalSize=47.8 K 2024-11-25T12:59:44,514 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26382852102148f4a936c279df67a892, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732539580854 2024-11-25T12:59:44,514 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15447463641a45b491208e08f896cae6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1732539580943 2024-11-25T12:59:44,514 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 483313348eab4a8e9ff723d02a9dcead, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732539581651 2024-11-25T12:59:44,514 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting b09eceb035c1467b8870b9d89f8b0929, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732539582926 2024-11-25T12:59:44,526 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6a5867ce2299ba674b3f042fd1fb41c6#C#compaction#401 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:59:44,526 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/b7f61d617cb94bf38de9aadcd548f2f4 is 50, key is test_row_0/C:col10/1732539583614/Put/seqid=0 2024-11-25T12:59:44,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742305_1481 (size=12595) 2024-11-25T12:59:44,564 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/b7f61d617cb94bf38de9aadcd548f2f4 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/b7f61d617cb94bf38de9aadcd548f2f4 2024-11-25T12:59:44,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:44,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56072 deadline: 1732539644560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:44,571 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:44,571 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:44,571 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:44,574 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6a5867ce2299ba674b3f042fd1fb41c6/C of 6a5867ce2299ba674b3f042fd1fb41c6 into b7f61d617cb94bf38de9aadcd548f2f4(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:59:44,574 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6a5867ce2299ba674b3f042fd1fb41c6: 2024-11-25T12:59:44,574 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., storeName=6a5867ce2299ba674b3f042fd1fb41c6/C, priority=12, startTime=1732539584327; duration=0sec 2024-11-25T12:59:44,574 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:59:44,574 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6a5867ce2299ba674b3f042fd1fb41c6:C 2024-11-25T12:59:44,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:44,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56100 deadline: 1732539644590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:44,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:44,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:44,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:44,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56124 deadline: 1732539644590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:44,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:44,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:44,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56108 deadline: 1732539644590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:44,598 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:44,598 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:44,598 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:44,598 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:44,598 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:44,598 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:44,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:44,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56082 deadline: 1732539644600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:44,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:44,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:44,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:44,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:44,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56072 deadline: 1732539644780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:44,784 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:44,784 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:44,784 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:44,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:44,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56100 deadline: 1732539644799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:44,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:44,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:44,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:44,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:44,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56108 deadline: 1732539644810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:44,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:44,816 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:44,816 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:44,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56124 deadline: 1732539644810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:44,816 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:44,816 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:44,816 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:44,816 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:44,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:44,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56082 deadline: 1732539644820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:44,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:44,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:44,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:44,830 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/3983032290e34afc8aeb0bb748ebe92a 2024-11-25T12:59:44,836 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/b95427c220764406bf3b4a2db4e9c46e is 50, key is test_row_0/B:col10/1732539583638/Put/seqid=0 2024-11-25T12:59:44,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742306_1482 (size=12151) 2024-11-25T12:59:44,843 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/b95427c220764406bf3b4a2db4e9c46e 2024-11-25T12:59:44,848 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/d867629f5f3f4f2ca43e483ae05325ee is 50, key is test_row_0/C:col10/1732539583638/Put/seqid=0 2024-11-25T12:59:44,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742307_1483 (size=12151) 2024-11-25T12:59:44,857 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/d867629f5f3f4f2ca43e483ae05325ee 2024-11-25T12:59:44,861 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/3983032290e34afc8aeb0bb748ebe92a as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/3983032290e34afc8aeb0bb748ebe92a 2024-11-25T12:59:44,865 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/3983032290e34afc8aeb0bb748ebe92a, entries=200, sequenceid=210, filesize=14.2 K 2024-11-25T12:59:44,866 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/b95427c220764406bf3b4a2db4e9c46e as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/b95427c220764406bf3b4a2db4e9c46e 2024-11-25T12:59:44,870 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/b95427c220764406bf3b4a2db4e9c46e, entries=150, sequenceid=210, filesize=11.9 K 2024-11-25T12:59:44,871 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/d867629f5f3f4f2ca43e483ae05325ee as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/d867629f5f3f4f2ca43e483ae05325ee 2024-11-25T12:59:44,876 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/d867629f5f3f4f2ca43e483ae05325ee, entries=150, sequenceid=210, filesize=11.9 K 2024-11-25T12:59:44,877 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=114.05 KB/116790 for 6a5867ce2299ba674b3f042fd1fb41c6 in 533ms, sequenceid=210, compaction requested=false 2024-11-25T12:59:44,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6a5867ce2299ba674b3f042fd1fb41c6: 2024-11-25T12:59:44,878 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/87c623cd79a342f7bd6acb4f73d6c004 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/87c623cd79a342f7bd6acb4f73d6c004 2024-11-25T12:59:44,883 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6a5867ce2299ba674b3f042fd1fb41c6/B of 6a5867ce2299ba674b3f042fd1fb41c6 into 87c623cd79a342f7bd6acb4f73d6c004(size=12.3 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:59:44,883 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6a5867ce2299ba674b3f042fd1fb41c6: 2024-11-25T12:59:44,883 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., storeName=6a5867ce2299ba674b3f042fd1fb41c6/B, priority=12, startTime=1732539584326; duration=0sec 2024-11-25T12:59:44,883 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:59:44,883 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6a5867ce2299ba674b3f042fd1fb41c6:B 2024-11-25T12:59:45,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 6a5867ce2299ba674b3f042fd1fb41c6 2024-11-25T12:59:45,094 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:59:45,094 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6a5867ce2299ba674b3f042fd1fb41c6 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-25T12:59:45,102 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/001f5af38ca24e0985f10fe419451bfb is 50, key is test_row_0/A:col10/1732539585091/Put/seqid=0 2024-11-25T12:59:45,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742308_1484 (size=14541) 2024-11-25T12:59:45,121 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/001f5af38ca24e0985f10fe419451bfb 2024-11-25T12:59:45,137 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/1776e69b72b647b696953594bde4294e is 50, key is test_row_0/B:col10/1732539585091/Put/seqid=0 2024-11-25T12:59:45,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:45,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56108 deadline: 1732539645148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:45,161 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:45,161 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:45,161 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:45,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:45,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56100 deadline: 1732539645158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:45,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:45,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:45,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:45,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:45,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56124 deadline: 1732539645161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:45,177 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:45,177 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:45,177 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:45,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:45,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56082 deadline: 1732539645161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:45,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:45,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:45,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:45,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:45,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56072 deadline: 1732539645161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:45,177 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:45,177 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:45,177 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:45,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742309_1485 (size=12151) 2024-11-25T12:59:45,188 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/1776e69b72b647b696953594bde4294e 2024-11-25T12:59:45,215 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/8932749270bc4fd38aaafad6076f8881 is 50, key is test_row_0/C:col10/1732539585091/Put/seqid=0 2024-11-25T12:59:45,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742310_1486 (size=12151) 2024-11-25T12:59:45,241 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/8932749270bc4fd38aaafad6076f8881 2024-11-25T12:59:45,248 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/001f5af38ca24e0985f10fe419451bfb as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/001f5af38ca24e0985f10fe419451bfb 2024-11-25T12:59:45,251 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/001f5af38ca24e0985f10fe419451bfb, entries=200, sequenceid=234, filesize=14.2 K 2024-11-25T12:59:45,252 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/1776e69b72b647b696953594bde4294e as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/1776e69b72b647b696953594bde4294e 2024-11-25T12:59:45,255 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/1776e69b72b647b696953594bde4294e, entries=150, sequenceid=234, filesize=11.9 K 2024-11-25T12:59:45,256 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/8932749270bc4fd38aaafad6076f8881 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/8932749270bc4fd38aaafad6076f8881 2024-11-25T12:59:45,258 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/8932749270bc4fd38aaafad6076f8881, entries=150, sequenceid=234, filesize=11.9 K 2024-11-25T12:59:45,260 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=87.22 KB/89310 for 6a5867ce2299ba674b3f042fd1fb41c6 in 166ms, sequenceid=234, compaction requested=true 2024-11-25T12:59:45,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6a5867ce2299ba674b3f042fd1fb41c6: 2024-11-25T12:59:45,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6a5867ce2299ba674b3f042fd1fb41c6:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:59:45,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:59:45,260 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:59:45,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6a5867ce2299ba674b3f042fd1fb41c6:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:59:45,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:59:45,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6a5867ce2299ba674b3f042fd1fb41c6:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:59:45,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-25T12:59:45,261 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41677 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:59:45,261 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1541): 6a5867ce2299ba674b3f042fd1fb41c6/A is initiating minor compaction (all files) 2024-11-25T12:59:45,261 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6a5867ce2299ba674b3f042fd1fb41c6/A in TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:45,261 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/8f2c51c193dc4b0b87f1f5cf2c9d28a4, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/3983032290e34afc8aeb0bb748ebe92a, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/001f5af38ca24e0985f10fe419451bfb] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp, totalSize=40.7 K 2024-11-25T12:59:45,261 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f2c51c193dc4b0b87f1f5cf2c9d28a4, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732539582926 2024-11-25T12:59:45,261 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3983032290e34afc8aeb0bb748ebe92a, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732539583638 2024-11-25T12:59:45,262 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 001f5af38ca24e0985f10fe419451bfb, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732539584423 2024-11-25T12:59:45,263 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:59:45,264 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:59:45,264 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): 6a5867ce2299ba674b3f042fd1fb41c6/B is initiating minor compaction (all files) 2024-11-25T12:59:45,264 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6a5867ce2299ba674b3f042fd1fb41c6/B in TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:45,264 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/87c623cd79a342f7bd6acb4f73d6c004, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/b95427c220764406bf3b4a2db4e9c46e, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/1776e69b72b647b696953594bde4294e] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp, totalSize=36.0 K 2024-11-25T12:59:45,264 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 87c623cd79a342f7bd6acb4f73d6c004, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732539582926 2024-11-25T12:59:45,265 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting b95427c220764406bf3b4a2db4e9c46e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732539583638 2024-11-25T12:59:45,265 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 1776e69b72b647b696953594bde4294e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732539584423 2024-11-25T12:59:45,272 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6a5867ce2299ba674b3f042fd1fb41c6#A#compaction#407 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:59:45,273 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/fbcb82c00bb04e51b1f72f2f1d0e66ff is 50, key is test_row_0/A:col10/1732539585091/Put/seqid=0 2024-11-25T12:59:45,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 6a5867ce2299ba674b3f042fd1fb41c6 2024-11-25T12:59:45,284 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:59:45,284 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6a5867ce2299ba674b3f042fd1fb41c6 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-25T12:59:45,293 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6a5867ce2299ba674b3f042fd1fb41c6#B#compaction#408 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:59:45,293 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/e75c17021add424486992224f943be3f is 50, key is test_row_0/B:col10/1732539585091/Put/seqid=0 2024-11-25T12:59:45,309 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/851c4b92b9bb48dc960affd9f61f174c is 50, key is test_row_0/A:col10/1732539585158/Put/seqid=0 2024-11-25T12:59:45,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742311_1487 (size=12697) 2024-11-25T12:59:45,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-25T12:59:45,320 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:59:45,320 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:59:45,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-25T12:59:45,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-25T12:59:45,328 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:59:45,329 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:59:45,329 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:59:45,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742312_1488 (size=12697) 2024-11-25T12:59:45,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:45,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56108 deadline: 1732539645340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:45,365 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:45,365 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:45,365 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:45,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:45,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56082 deadline: 1732539645353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:45,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:45,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:45,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:45,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:45,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56072 deadline: 1732539645353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:45,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:45,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:45,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:45,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742313_1489 (size=14541) 2024-11-25T12:59:45,376 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/851c4b92b9bb48dc960affd9f61f174c 2024-11-25T12:59:45,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:45,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56124 deadline: 1732539645365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:45,381 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:45,381 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:45,381 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:45,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:45,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56100 deadline: 1732539645369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:45,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:45,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:45,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:45,393 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/99ec25698a104627862856ef539b1565 is 50, key is test_row_0/B:col10/1732539585158/Put/seqid=0 2024-11-25T12:59:45,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-25T12:59:45,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742314_1490 (size=12151) 2024-11-25T12:59:45,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:45,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56108 deadline: 1732539645470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:45,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:45,481 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:45,481 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:45,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56082 deadline: 1732539645470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:45,481 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:45,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:45,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:45,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:45,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:45,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56072 deadline: 1732539645470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:45,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:45,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:45,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:45,491 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-25T12:59:45,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:45,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. as already flushing 2024-11-25T12:59:45,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:45,494 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:45,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:45,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:45,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:45,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56124 deadline: 1732539645490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:45,502 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:45,502 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:45,502 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:45,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:45,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56100 deadline: 1732539645490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:45,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:45,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:45,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:45,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-25T12:59:45,651 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-25T12:59:45,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:45,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. as already flushing 2024-11-25T12:59:45,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:45,652 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:45,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:45,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:45,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:45,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56108 deadline: 1732539645691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:45,695 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:45,696 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:45,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:45,696 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:45,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56082 deadline: 1732539645692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:45,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:45,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:45,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:45,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:45,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56072 deadline: 1732539645692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:45,697 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:45,697 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:45,697 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:45,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:45,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56100 deadline: 1732539645710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:45,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:45,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:45,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:45,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:45,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56124 deadline: 1732539645711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:45,725 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:45,725 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:45,725 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:45,752 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/fbcb82c00bb04e51b1f72f2f1d0e66ff as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/fbcb82c00bb04e51b1f72f2f1d0e66ff 2024-11-25T12:59:45,764 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6a5867ce2299ba674b3f042fd1fb41c6/A of 6a5867ce2299ba674b3f042fd1fb41c6 into fbcb82c00bb04e51b1f72f2f1d0e66ff(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:59:45,764 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6a5867ce2299ba674b3f042fd1fb41c6: 2024-11-25T12:59:45,764 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., storeName=6a5867ce2299ba674b3f042fd1fb41c6/A, priority=13, startTime=1732539585260; duration=0sec 2024-11-25T12:59:45,764 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:59:45,764 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6a5867ce2299ba674b3f042fd1fb41c6:A 2024-11-25T12:59:45,764 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:59:45,768 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:59:45,768 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1541): 6a5867ce2299ba674b3f042fd1fb41c6/C is initiating minor compaction (all files) 2024-11-25T12:59:45,768 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6a5867ce2299ba674b3f042fd1fb41c6/C in TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:45,768 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/b7f61d617cb94bf38de9aadcd548f2f4, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/d867629f5f3f4f2ca43e483ae05325ee, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/8932749270bc4fd38aaafad6076f8881] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp, totalSize=36.0 K 2024-11-25T12:59:45,769 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7f61d617cb94bf38de9aadcd548f2f4, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732539582926 2024-11-25T12:59:45,771 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting d867629f5f3f4f2ca43e483ae05325ee, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732539583638 2024-11-25T12:59:45,775 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8932749270bc4fd38aaafad6076f8881, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732539584423 2024-11-25T12:59:45,795 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/e75c17021add424486992224f943be3f as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/e75c17021add424486992224f943be3f 2024-11-25T12:59:45,808 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-25T12:59:45,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:45,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. as already flushing 2024-11-25T12:59:45,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:45,811 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:45,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:45,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:45,819 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6a5867ce2299ba674b3f042fd1fb41c6/B of 6a5867ce2299ba674b3f042fd1fb41c6 into e75c17021add424486992224f943be3f(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:59:45,819 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6a5867ce2299ba674b3f042fd1fb41c6: 2024-11-25T12:59:45,820 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., storeName=6a5867ce2299ba674b3f042fd1fb41c6/B, priority=13, startTime=1732539585260; duration=0sec 2024-11-25T12:59:45,820 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:59:45,820 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6a5867ce2299ba674b3f042fd1fb41c6:B 2024-11-25T12:59:45,835 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6a5867ce2299ba674b3f042fd1fb41c6#C#compaction#411 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:59:45,835 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/c9130b086016474f972e01930e5fd492 is 50, key is test_row_0/C:col10/1732539585091/Put/seqid=0 2024-11-25T12:59:45,842 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/99ec25698a104627862856ef539b1565 2024-11-25T12:59:45,873 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/bafa155d6c8148239c44b6b4d7565f1f is 50, key is test_row_0/C:col10/1732539585158/Put/seqid=0 2024-11-25T12:59:45,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742315_1491 (size=12697) 2024-11-25T12:59:45,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742316_1492 (size=12151) 2024-11-25T12:59:45,917 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/bafa155d6c8148239c44b6b4d7565f1f 2024-11-25T12:59:45,929 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/851c4b92b9bb48dc960affd9f61f174c as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/851c4b92b9bb48dc960affd9f61f174c 2024-11-25T12:59:45,937 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/851c4b92b9bb48dc960affd9f61f174c, entries=200, sequenceid=251, filesize=14.2 K 2024-11-25T12:59:45,938 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/99ec25698a104627862856ef539b1565 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/99ec25698a104627862856ef539b1565 2024-11-25T12:59:45,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-25T12:59:45,951 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/99ec25698a104627862856ef539b1565, entries=150, sequenceid=251, filesize=11.9 K 2024-11-25T12:59:45,952 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/bafa155d6c8148239c44b6b4d7565f1f as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/bafa155d6c8148239c44b6b4d7565f1f 2024-11-25T12:59:45,957 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/bafa155d6c8148239c44b6b4d7565f1f, entries=150, sequenceid=251, filesize=11.9 K 2024-11-25T12:59:45,959 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=114.05 KB/116790 for 6a5867ce2299ba674b3f042fd1fb41c6 in 675ms, sequenceid=251, compaction requested=false 2024-11-25T12:59:45,959 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6a5867ce2299ba674b3f042fd1fb41c6: 2024-11-25T12:59:45,970 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-25T12:59:45,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:45,971 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 6a5867ce2299ba674b3f042fd1fb41c6 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-25T12:59:45,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/15a204ce13254ac78a9846a6ef044d83 is 50, key is test_row_0/A:col10/1732539585340/Put/seqid=0 2024-11-25T12:59:46,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742317_1493 (size=12301) 2024-11-25T12:59:46,017 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/15a204ce13254ac78a9846a6ef044d83 2024-11-25T12:59:46,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. as already flushing 2024-11-25T12:59:46,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 6a5867ce2299ba674b3f042fd1fb41c6 2024-11-25T12:59:46,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/20acd0ae2e5940c883fba12c16b5bfd5 is 50, key is test_row_0/B:col10/1732539585340/Put/seqid=0 2024-11-25T12:59:46,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742318_1494 (size=12301) 2024-11-25T12:59:46,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:46,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56108 deadline: 1732539646084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:46,089 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:46,089 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:46,089 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:46,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:46,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56082 deadline: 1732539646087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:46,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:46,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:46,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56124 deadline: 1732539646087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:46,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:46,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:46,103 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:46,103 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:46,103 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:46,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:46,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56100 deadline: 1732539646088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:46,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:46,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:46,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:46,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:46,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56072 deadline: 1732539646090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:46,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:46,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:46,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:46,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:46,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56108 deadline: 1732539646200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:46,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:46,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:46,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:46,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:46,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56082 deadline: 1732539646216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:46,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:46,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56124 deadline: 1732539646217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:46,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:46,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56100 deadline: 1732539646217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:46,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:46,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:46,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56072 deadline: 1732539646217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:46,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:46,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:46,227 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:46,227 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:46,227 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:46,228 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:46,228 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:46,228 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:46,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:46,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:46,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:46,305 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/c9130b086016474f972e01930e5fd492 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/c9130b086016474f972e01930e5fd492 2024-11-25T12:59:46,310 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6a5867ce2299ba674b3f042fd1fb41c6/C of 6a5867ce2299ba674b3f042fd1fb41c6 into c9130b086016474f972e01930e5fd492(size=12.4 K), total size for store is 24.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:59:46,310 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6a5867ce2299ba674b3f042fd1fb41c6: 2024-11-25T12:59:46,310 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., storeName=6a5867ce2299ba674b3f042fd1fb41c6/C, priority=13, startTime=1732539585260; duration=0sec 2024-11-25T12:59:46,310 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:59:46,310 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6a5867ce2299ba674b3f042fd1fb41c6:C 2024-11-25T12:59:46,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:46,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56108 deadline: 1732539646422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:46,432 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:46,432 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:46,433 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:46,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:46,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56100 deadline: 1732539646430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:46,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:46,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:46,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:46,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56072 deadline: 1732539646430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:46,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:46,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:46,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:46,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:46,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:46,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56082 deadline: 1732539646430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:46,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:46,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:46,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:46,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:46,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56124 deadline: 1732539646442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:46,457 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:46,457 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:46,457 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:46,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-25T12:59:46,477 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/20acd0ae2e5940c883fba12c16b5bfd5 2024-11-25T12:59:46,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/2d68fd4b7d9c46afbc7a2cc324a5f9d2 is 50, key is test_row_0/C:col10/1732539585340/Put/seqid=0 2024-11-25T12:59:46,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742319_1495 (size=12301) 2024-11-25T12:59:46,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:46,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56108 deadline: 1732539646740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:46,746 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:46,746 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:46,747 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:46,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:46,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56100 deadline: 1732539646749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:46,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:46,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56082 deadline: 1732539646750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:46,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:46,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:46,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56072 deadline: 1732539646750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:46,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:46,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:46,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:46,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:46,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:46,759 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:46,759 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:46,759 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:46,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:46,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56124 deadline: 1732539646760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:46,765 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:46,765 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:46,765 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:46,941 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/2d68fd4b7d9c46afbc7a2cc324a5f9d2 2024-11-25T12:59:46,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/15a204ce13254ac78a9846a6ef044d83 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/15a204ce13254ac78a9846a6ef044d83 2024-11-25T12:59:46,957 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/15a204ce13254ac78a9846a6ef044d83, entries=150, sequenceid=273, filesize=12.0 K 2024-11-25T12:59:46,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/20acd0ae2e5940c883fba12c16b5bfd5 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/20acd0ae2e5940c883fba12c16b5bfd5 2024-11-25T12:59:46,969 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/20acd0ae2e5940c883fba12c16b5bfd5, entries=150, sequenceid=273, filesize=12.0 K 2024-11-25T12:59:46,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/2d68fd4b7d9c46afbc7a2cc324a5f9d2 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/2d68fd4b7d9c46afbc7a2cc324a5f9d2 2024-11-25T12:59:46,973 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/2d68fd4b7d9c46afbc7a2cc324a5f9d2, entries=150, sequenceid=273, filesize=12.0 K 2024-11-25T12:59:46,975 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 6a5867ce2299ba674b3f042fd1fb41c6 in 1004ms, sequenceid=273, compaction requested=true 2024-11-25T12:59:46,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 6a5867ce2299ba674b3f042fd1fb41c6: 2024-11-25T12:59:46,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:46,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-25T12:59:46,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-25T12:59:46,983 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-25T12:59:46,983 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6480 sec 2024-11-25T12:59:47,009 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.6790 sec 2024-11-25T12:59:47,258 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:59:47,258 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6a5867ce2299ba674b3f042fd1fb41c6 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-25T12:59:47,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 6a5867ce2299ba674b3f042fd1fb41c6 2024-11-25T12:59:47,290 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/bb5609380cd04ed482d5e9b1e1ab751a is 50, key is test_row_0/A:col10/1732539587257/Put/seqid=0 2024-11-25T12:59:47,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742320_1496 (size=14741) 2024-11-25T12:59:47,325 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/bb5609380cd04ed482d5e9b1e1ab751a 2024-11-25T12:59:47,352 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/b3ef34258c3d42cfbe033d41b764de02 is 50, key is test_row_0/B:col10/1732539587257/Put/seqid=0 2024-11-25T12:59:47,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742321_1497 (size=12301) 2024-11-25T12:59:47,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:47,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56072 deadline: 1732539647369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:47,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:47,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:47,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:47,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:47,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56124 deadline: 1732539647369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:47,384 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:47,384 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:47,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:47,384 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:47,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56100 deadline: 1732539647370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:47,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:47,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:47,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:47,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:47,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56108 deadline: 1732539647371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:47,384 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:47,384 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:47,385 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:47,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:47,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56082 deadline: 1732539647383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:47,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:47,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:47,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:47,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-25T12:59:47,470 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:59:47,471 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:59:47,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-25T12:59:47,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-25T12:59:47,472 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:59:47,473 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:59:47,473 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:59:47,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:47,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56072 deadline: 1732539647489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:47,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:47,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56124 deadline: 1732539647490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:47,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:47,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:47,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:47,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:47,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56100 deadline: 1732539647490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:47,498 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:47,498 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:47,498 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:47,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:47,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:47,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:47,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56108 deadline: 1732539647490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:47,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:47,498 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:47,498 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:47,498 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:47,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:47,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56082 deadline: 1732539647500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:47,509 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:47,509 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:47,509 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:47,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-25T12:59:47,627 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-25T12:59:47,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:47,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. as already flushing 2024-11-25T12:59:47,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:47,627 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:47,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:47,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:47,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:47,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56072 deadline: 1732539647700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:47,701 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:47,701 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:47,701 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:47,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:47,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56108 deadline: 1732539647710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:47,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:47,713 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:47,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56124 deadline: 1732539647710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:47,713 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:47,713 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:47,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:47,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56100 deadline: 1732539647710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:47,714 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:47,714 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:47,714 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:47,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:47,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:47,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:47,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:47,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56082 deadline: 1732539647720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:47,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:47,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:47,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:47,765 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/b3ef34258c3d42cfbe033d41b764de02 2024-11-25T12:59:47,770 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/618b4b25ff944c31a50eaa50f64877ee is 50, key is test_row_0/C:col10/1732539587257/Put/seqid=0 2024-11-25T12:59:47,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742322_1498 (size=12301) 2024-11-25T12:59:47,779 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/618b4b25ff944c31a50eaa50f64877ee 2024-11-25T12:59:47,780 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-25T12:59:47,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:47,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. as already flushing 2024-11-25T12:59:47,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:47,781 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:47,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:47,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:47,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-25T12:59:47,799 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/bb5609380cd04ed482d5e9b1e1ab751a as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/bb5609380cd04ed482d5e9b1e1ab751a 2024-11-25T12:59:47,803 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/bb5609380cd04ed482d5e9b1e1ab751a, entries=200, sequenceid=291, filesize=14.4 K 2024-11-25T12:59:47,810 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/b3ef34258c3d42cfbe033d41b764de02 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/b3ef34258c3d42cfbe033d41b764de02 2024-11-25T12:59:47,821 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/b3ef34258c3d42cfbe033d41b764de02, entries=150, sequenceid=291, filesize=12.0 K 2024-11-25T12:59:47,825 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/618b4b25ff944c31a50eaa50f64877ee as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/618b4b25ff944c31a50eaa50f64877ee 2024-11-25T12:59:47,840 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/618b4b25ff944c31a50eaa50f64877ee, entries=150, sequenceid=291, filesize=12.0 K 2024-11-25T12:59:47,841 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 6a5867ce2299ba674b3f042fd1fb41c6 in 583ms, sequenceid=291, compaction requested=true 2024-11-25T12:59:47,841 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6a5867ce2299ba674b3f042fd1fb41c6: 2024-11-25T12:59:47,842 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:59:47,843 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54280 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:59:47,843 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1541): 6a5867ce2299ba674b3f042fd1fb41c6/A is initiating minor compaction (all files) 2024-11-25T12:59:47,843 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6a5867ce2299ba674b3f042fd1fb41c6/A in TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:47,843 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/fbcb82c00bb04e51b1f72f2f1d0e66ff, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/851c4b92b9bb48dc960affd9f61f174c, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/15a204ce13254ac78a9846a6ef044d83, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/bb5609380cd04ed482d5e9b1e1ab751a] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp, totalSize=53.0 K 2024-11-25T12:59:47,843 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting fbcb82c00bb04e51b1f72f2f1d0e66ff, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732539584423 2024-11-25T12:59:47,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6a5867ce2299ba674b3f042fd1fb41c6:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:59:47,843 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 851c4b92b9bb48dc960affd9f61f174c, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732539585141 2024-11-25T12:59:47,844 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15a204ce13254ac78a9846a6ef044d83, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1732539585340 2024-11-25T12:59:47,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:59:47,844 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb5609380cd04ed482d5e9b1e1ab751a, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732539586082 2024-11-25T12:59:47,845 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:59:47,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6a5867ce2299ba674b3f042fd1fb41c6:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:59:47,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:59:47,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6a5867ce2299ba674b3f042fd1fb41c6:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:59:47,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:59:47,851 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49450 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:59:47,851 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): 6a5867ce2299ba674b3f042fd1fb41c6/B is initiating minor compaction (all files) 2024-11-25T12:59:47,851 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6a5867ce2299ba674b3f042fd1fb41c6/B in TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:47,851 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/e75c17021add424486992224f943be3f, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/99ec25698a104627862856ef539b1565, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/20acd0ae2e5940c883fba12c16b5bfd5, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/b3ef34258c3d42cfbe033d41b764de02] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp, totalSize=48.3 K 2024-11-25T12:59:47,855 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting e75c17021add424486992224f943be3f, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732539584423 2024-11-25T12:59:47,856 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 99ec25698a104627862856ef539b1565, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732539585157 2024-11-25T12:59:47,856 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 20acd0ae2e5940c883fba12c16b5bfd5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1732539585340 2024-11-25T12:59:47,857 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting b3ef34258c3d42cfbe033d41b764de02, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732539586082 2024-11-25T12:59:47,866 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6a5867ce2299ba674b3f042fd1fb41c6#A#compaction#419 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:59:47,866 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/5075ebac0a99454d8f2f7e3fdbf73a0e is 50, key is test_row_0/A:col10/1732539587257/Put/seqid=0 2024-11-25T12:59:47,901 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6a5867ce2299ba674b3f042fd1fb41c6#B#compaction#420 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:59:47,901 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/3c04192c654d4b20a34072f3d52f592c is 50, key is test_row_0/B:col10/1732539587257/Put/seqid=0 2024-11-25T12:59:47,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742323_1499 (size=12983) 2024-11-25T12:59:47,933 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-25T12:59:47,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:47,934 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 6a5867ce2299ba674b3f042fd1fb41c6 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-25T12:59:47,953 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/5075ebac0a99454d8f2f7e3fdbf73a0e as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/5075ebac0a99454d8f2f7e3fdbf73a0e 2024-11-25T12:59:47,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742324_1500 (size=12983) 2024-11-25T12:59:47,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/0cffcfd00b5f494b857dcf50e7156116 is 50, key is test_row_0/A:col10/1732539587370/Put/seqid=0 2024-11-25T12:59:47,995 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/3c04192c654d4b20a34072f3d52f592c as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/3c04192c654d4b20a34072f3d52f592c 2024-11-25T12:59:47,998 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6a5867ce2299ba674b3f042fd1fb41c6/A of 6a5867ce2299ba674b3f042fd1fb41c6 into 5075ebac0a99454d8f2f7e3fdbf73a0e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:59:47,998 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6a5867ce2299ba674b3f042fd1fb41c6: 2024-11-25T12:59:47,998 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., storeName=6a5867ce2299ba674b3f042fd1fb41c6/A, priority=12, startTime=1732539587841; duration=0sec 2024-11-25T12:59:47,998 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:59:47,998 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6a5867ce2299ba674b3f042fd1fb41c6:A 2024-11-25T12:59:47,998 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:59:48,000 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49450 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:59:48,000 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1541): 6a5867ce2299ba674b3f042fd1fb41c6/C is initiating minor compaction (all files) 2024-11-25T12:59:48,000 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6a5867ce2299ba674b3f042fd1fb41c6/C in TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:48,000 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/c9130b086016474f972e01930e5fd492, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/bafa155d6c8148239c44b6b4d7565f1f, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/2d68fd4b7d9c46afbc7a2cc324a5f9d2, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/618b4b25ff944c31a50eaa50f64877ee] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp, totalSize=48.3 K 2024-11-25T12:59:48,000 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9130b086016474f972e01930e5fd492, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732539584423 2024-11-25T12:59:48,000 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting bafa155d6c8148239c44b6b4d7565f1f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732539585157 2024-11-25T12:59:48,001 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d68fd4b7d9c46afbc7a2cc324a5f9d2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1732539585340 2024-11-25T12:59:48,001 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 618b4b25ff944c31a50eaa50f64877ee, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732539586082 2024-11-25T12:59:48,004 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6a5867ce2299ba674b3f042fd1fb41c6/B of 6a5867ce2299ba674b3f042fd1fb41c6 into 3c04192c654d4b20a34072f3d52f592c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:59:48,004 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6a5867ce2299ba674b3f042fd1fb41c6: 2024-11-25T12:59:48,004 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., storeName=6a5867ce2299ba674b3f042fd1fb41c6/B, priority=12, startTime=1732539587845; duration=0sec 2024-11-25T12:59:48,004 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:59:48,004 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6a5867ce2299ba674b3f042fd1fb41c6:B 2024-11-25T12:59:48,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742325_1501 (size=12301) 2024-11-25T12:59:48,020 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. as already flushing 2024-11-25T12:59:48,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 6a5867ce2299ba674b3f042fd1fb41c6 2024-11-25T12:59:48,026 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/0cffcfd00b5f494b857dcf50e7156116 2024-11-25T12:59:48,034 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6a5867ce2299ba674b3f042fd1fb41c6#C#compaction#422 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:59:48,034 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/890af16b178545b2b0e3c5894cee5af4 is 50, key is test_row_0/C:col10/1732539587257/Put/seqid=0 2024-11-25T12:59:48,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/ddbd42152a114665b1f4f90479e22376 is 50, key is test_row_0/B:col10/1732539587370/Put/seqid=0 2024-11-25T12:59:48,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742326_1502 (size=12983) 2024-11-25T12:59:48,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:48,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56082 deadline: 1732539648056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:48,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:48,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:48,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:48,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742327_1503 (size=12301) 2024-11-25T12:59:48,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:48,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56124 deadline: 1732539648072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:48,089 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:48,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:48,089 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:48,089 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:48,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56072 deadline: 1732539648072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:48,089 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:48,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:48,089 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:48,089 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:48,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56100 deadline: 1732539648073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:48,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:48,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:48,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:48,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:48,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56108 deadline: 1732539648073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:48,090 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:48,090 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:48,090 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:48,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-25T12:59:48,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:48,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56082 deadline: 1732539648182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:48,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:48,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:48,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:48,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:48,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56108 deadline: 1732539648200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:48,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:48,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56124 deadline: 1732539648200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:48,209 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:48,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:48,209 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:48,209 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:48,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56100 deadline: 1732539648200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:48,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:48,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:48,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:48,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:48,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56072 deadline: 1732539648200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:48,210 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:48,210 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:48,210 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:48,212 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:48,212 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:48,212 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:48,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:48,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56082 deadline: 1732539648395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:48,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:48,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:48,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:48,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:48,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56072 deadline: 1732539648422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:48,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:48,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:48,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:48,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:48,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56108 deadline: 1732539648422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:48,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:48,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56124 deadline: 1732539648422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:48,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:48,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56100 deadline: 1732539648422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:48,438 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:48,438 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:48,438 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:48,438 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:48,438 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:48,438 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:48,441 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:48,441 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:48,441 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:48,481 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/ddbd42152a114665b1f4f90479e22376 2024-11-25T12:59:48,506 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/890af16b178545b2b0e3c5894cee5af4 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/890af16b178545b2b0e3c5894cee5af4 2024-11-25T12:59:48,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/cce28a5125274799a9a0c746b4385726 is 50, key is test_row_0/C:col10/1732539587370/Put/seqid=0 2024-11-25T12:59:48,516 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6a5867ce2299ba674b3f042fd1fb41c6/C of 6a5867ce2299ba674b3f042fd1fb41c6 into 890af16b178545b2b0e3c5894cee5af4(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:59:48,516 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6a5867ce2299ba674b3f042fd1fb41c6: 2024-11-25T12:59:48,516 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., storeName=6a5867ce2299ba674b3f042fd1fb41c6/C, priority=12, startTime=1732539587846; duration=0sec 2024-11-25T12:59:48,516 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:59:48,517 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6a5867ce2299ba674b3f042fd1fb41c6:C 2024-11-25T12:59:48,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742328_1504 (size=12301) 2024-11-25T12:59:48,551 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/cce28a5125274799a9a0c746b4385726 2024-11-25T12:59:48,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/0cffcfd00b5f494b857dcf50e7156116 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/0cffcfd00b5f494b857dcf50e7156116 2024-11-25T12:59:48,560 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/0cffcfd00b5f494b857dcf50e7156116, entries=150, sequenceid=310, filesize=12.0 K 2024-11-25T12:59:48,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/ddbd42152a114665b1f4f90479e22376 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/ddbd42152a114665b1f4f90479e22376 2024-11-25T12:59:48,564 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/ddbd42152a114665b1f4f90479e22376, entries=150, sequenceid=310, filesize=12.0 K 2024-11-25T12:59:48,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/cce28a5125274799a9a0c746b4385726 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/cce28a5125274799a9a0c746b4385726 2024-11-25T12:59:48,568 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/cce28a5125274799a9a0c746b4385726, entries=150, sequenceid=310, filesize=12.0 K 2024-11-25T12:59:48,568 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=100.63 KB/103050 for 6a5867ce2299ba674b3f042fd1fb41c6 in 634ms, sequenceid=310, compaction requested=false 2024-11-25T12:59:48,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 6a5867ce2299ba674b3f042fd1fb41c6: 2024-11-25T12:59:48,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:48,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-25T12:59:48,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-25T12:59:48,579 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-25T12:59:48,579 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1040 sec 2024-11-25T12:59:48,580 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.1090 sec 2024-11-25T12:59:48,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-25T12:59:48,610 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:59:48,610 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:59:48,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-25T12:59:48,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-25T12:59:48,616 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:59:48,616 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:59:48,616 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:59:48,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 6a5867ce2299ba674b3f042fd1fb41c6 2024-11-25T12:59:48,715 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:59:48,715 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6a5867ce2299ba674b3f042fd1fb41c6 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-25T12:59:48,719 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/547507aa9ea54972b3339666e82d7de2 is 50, key is test_row_0/A:col10/1732539588072/Put/seqid=0 2024-11-25T12:59:48,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-25T12:59:48,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742329_1505 (size=14741) 2024-11-25T12:59:48,769 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-25T12:59:48,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:48,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. as already flushing 2024-11-25T12:59:48,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:48,769 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:48,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:48,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:48,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:48,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56072 deadline: 1732539648765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:48,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:48,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:48,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:48,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:48,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56082 deadline: 1732539648766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:48,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:48,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:48,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:48,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56100 deadline: 1732539648767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:48,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:48,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:48,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:48,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:48,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56108 deadline: 1732539648768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:48,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:48,780 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:48,780 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:48,780 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:48,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:48,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56124 deadline: 1732539648770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:48,782 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:48,782 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:48,782 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:48,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:48,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56072 deadline: 1732539648880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:48,885 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:48,885 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:48,885 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:48,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:48,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56082 deadline: 1732539648890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:48,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:48,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:48,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:48,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:48,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56108 deadline: 1732539648890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:48,898 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:48,898 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:48,898 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:48,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:48,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56100 deadline: 1732539648890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:48,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:48,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56124 deadline: 1732539648891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:48,900 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:48,900 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:48,900 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:48,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:48,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:48,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:48,922 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-25T12:59:48,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:48,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. as already flushing 2024-11-25T12:59:48,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:48,922 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:48,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:48,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:48,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-25T12:59:49,074 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-25T12:59:49,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:49,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. as already flushing 2024-11-25T12:59:49,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:49,075 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:49,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:49,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:49,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:49,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56072 deadline: 1732539649091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:49,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:49,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:49,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:49,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:49,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56108 deadline: 1732539649112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:49,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:49,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:49,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:49,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:49,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56082 deadline: 1732539649112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:49,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:49,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:49,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:49,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:49,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56124 deadline: 1732539649112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:49,119 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:49,119 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:49,119 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:49,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:49,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56100 deadline: 1732539649112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:49,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:49,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:49,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:49,129 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/547507aa9ea54972b3339666e82d7de2 2024-11-25T12:59:49,136 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/70107ca6483747f498c66a4bced9f6b1 is 50, key is test_row_0/B:col10/1732539588072/Put/seqid=0 2024-11-25T12:59:49,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742330_1506 (size=12301) 2024-11-25T12:59:49,228 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-25T12:59:49,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:49,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. as already flushing 2024-11-25T12:59:49,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:49,229 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:49,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:49,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:49,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-25T12:59:49,385 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-25T12:59:49,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:49,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. as already flushing 2024-11-25T12:59:49,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:49,385 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:49,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:49,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:49,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:49,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56072 deadline: 1732539649414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:49,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:49,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:49,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:49,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:49,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56124 deadline: 1732539649430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:49,439 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:49,440 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:49,440 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:49,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:49,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56100 deadline: 1732539649431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:49,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:49,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:49,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:49,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56108 deadline: 1732539649431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:49,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:49,446 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:49,446 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:49,446 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:49,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:49,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56082 deadline: 1732539649431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:49,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:49,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:49,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:49,543 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-25T12:59:49,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:49,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. as already flushing 2024-11-25T12:59:49,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:49,544 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:49,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:49,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:49,612 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/70107ca6483747f498c66a4bced9f6b1 2024-11-25T12:59:49,641 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/bf87499974b1478a97e69c1fa6af8ff9 is 50, key is test_row_0/C:col10/1732539588072/Put/seqid=0 2024-11-25T12:59:49,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742331_1507 (size=12301) 2024-11-25T12:59:49,697 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-25T12:59:49,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:49,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. as already flushing 2024-11-25T12:59:49,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:49,698 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:49,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:49,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:49,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-25T12:59:49,851 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-25T12:59:49,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:49,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. as already flushing 2024-11-25T12:59:49,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:49,852 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:49,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:49,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:49,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:49,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56072 deadline: 1732539649930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:49,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:49,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:49,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:49,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:49,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56124 deadline: 1732539649950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:49,957 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:49,957 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:49,957 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:49,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:49,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56100 deadline: 1732539649959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:49,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:49,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:49,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:49,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:49,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56108 deadline: 1732539649959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:49,972 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:49,972 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:49,972 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:49,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:49,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56082 deadline: 1732539649960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:49,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:49,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:49,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:50,008 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-25T12:59:50,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:50,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. as already flushing 2024-11-25T12:59:50,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:50,008 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:50,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:50,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:50,095 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/bf87499974b1478a97e69c1fa6af8ff9 2024-11-25T12:59:50,099 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/547507aa9ea54972b3339666e82d7de2 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/547507aa9ea54972b3339666e82d7de2 2024-11-25T12:59:50,105 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/547507aa9ea54972b3339666e82d7de2, entries=200, sequenceid=332, filesize=14.4 K 2024-11-25T12:59:50,106 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/70107ca6483747f498c66a4bced9f6b1 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/70107ca6483747f498c66a4bced9f6b1 2024-11-25T12:59:50,109 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/70107ca6483747f498c66a4bced9f6b1, entries=150, sequenceid=332, filesize=12.0 K 2024-11-25T12:59:50,119 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/bf87499974b1478a97e69c1fa6af8ff9 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/bf87499974b1478a97e69c1fa6af8ff9 2024-11-25T12:59:50,123 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/bf87499974b1478a97e69c1fa6af8ff9, entries=150, sequenceid=332, filesize=12.0 K 2024-11-25T12:59:50,124 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 6a5867ce2299ba674b3f042fd1fb41c6 in 1409ms, sequenceid=332, compaction requested=true 2024-11-25T12:59:50,124 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6a5867ce2299ba674b3f042fd1fb41c6: 2024-11-25T12:59:50,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6a5867ce2299ba674b3f042fd1fb41c6:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:59:50,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:59:50,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6a5867ce2299ba674b3f042fd1fb41c6:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:59:50,124 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:59:50,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:59:50,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6a5867ce2299ba674b3f042fd1fb41c6:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:59:50,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-25T12:59:50,124 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:59:50,125 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:59:50,125 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): 6a5867ce2299ba674b3f042fd1fb41c6/A is initiating minor compaction (all files) 2024-11-25T12:59:50,125 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6a5867ce2299ba674b3f042fd1fb41c6/A in TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:50,125 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/5075ebac0a99454d8f2f7e3fdbf73a0e, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/0cffcfd00b5f494b857dcf50e7156116, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/547507aa9ea54972b3339666e82d7de2] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp, totalSize=39.1 K 2024-11-25T12:59:50,126 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 5075ebac0a99454d8f2f7e3fdbf73a0e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732539586082 2024-11-25T12:59:50,126 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:59:50,126 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1541): 6a5867ce2299ba674b3f042fd1fb41c6/B is initiating minor compaction (all files) 2024-11-25T12:59:50,126 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6a5867ce2299ba674b3f042fd1fb41c6/B in TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:50,126 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/3c04192c654d4b20a34072f3d52f592c, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/ddbd42152a114665b1f4f90479e22376, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/70107ca6483747f498c66a4bced9f6b1] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp, totalSize=36.7 K 2024-11-25T12:59:50,126 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 0cffcfd00b5f494b857dcf50e7156116, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1732539587301 2024-11-25T12:59:50,127 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c04192c654d4b20a34072f3d52f592c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732539586082 2024-11-25T12:59:50,127 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 547507aa9ea54972b3339666e82d7de2, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1732539588037 2024-11-25T12:59:50,127 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting ddbd42152a114665b1f4f90479e22376, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1732539587301 2024-11-25T12:59:50,129 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70107ca6483747f498c66a4bced9f6b1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1732539588037 2024-11-25T12:59:50,149 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6a5867ce2299ba674b3f042fd1fb41c6#A#compaction#428 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:59:50,150 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/721ff44d3faf435ca6582e4949379ab3 is 50, key is test_row_0/A:col10/1732539588072/Put/seqid=0 2024-11-25T12:59:50,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-25T12:59:50,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:50,162 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 6a5867ce2299ba674b3f042fd1fb41c6 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-25T12:59:50,165 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6a5867ce2299ba674b3f042fd1fb41c6#B#compaction#429 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:59:50,166 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/318b803f235c470cbcdd7f83d8003a11 is 50, key is test_row_0/B:col10/1732539588072/Put/seqid=0 2024-11-25T12:59:50,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/f9ef43217b5e4ff6b628178cd496cbbc is 50, key is test_row_0/A:col10/1732539588766/Put/seqid=0 2024-11-25T12:59:50,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742332_1508 (size=13085) 2024-11-25T12:59:50,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742334_1510 (size=12301) 2024-11-25T12:59:50,286 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/f9ef43217b5e4ff6b628178cd496cbbc 2024-11-25T12:59:50,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742333_1509 (size=13085) 2024-11-25T12:59:50,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/b5076c7407fc4d9bbb9f52b0c05ec7b3 is 50, key is test_row_0/B:col10/1732539588766/Put/seqid=0 2024-11-25T12:59:50,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742335_1511 (size=12301) 2024-11-25T12:59:50,347 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/b5076c7407fc4d9bbb9f52b0c05ec7b3 2024-11-25T12:59:50,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/b568fc5612c249d9a8200e786d0547ca is 50, key is test_row_0/C:col10/1732539588766/Put/seqid=0 2024-11-25T12:59:50,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742336_1512 (size=12301) 2024-11-25T12:59:50,655 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/721ff44d3faf435ca6582e4949379ab3 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/721ff44d3faf435ca6582e4949379ab3 2024-11-25T12:59:50,665 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6a5867ce2299ba674b3f042fd1fb41c6/A of 6a5867ce2299ba674b3f042fd1fb41c6 into 721ff44d3faf435ca6582e4949379ab3(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:59:50,665 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6a5867ce2299ba674b3f042fd1fb41c6: 2024-11-25T12:59:50,665 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., storeName=6a5867ce2299ba674b3f042fd1fb41c6/A, priority=13, startTime=1732539590124; duration=0sec 2024-11-25T12:59:50,665 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:59:50,665 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6a5867ce2299ba674b3f042fd1fb41c6:A 2024-11-25T12:59:50,665 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:59:50,666 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:59:50,666 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): 6a5867ce2299ba674b3f042fd1fb41c6/C is initiating minor compaction (all files) 2024-11-25T12:59:50,666 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6a5867ce2299ba674b3f042fd1fb41c6/C in TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:50,666 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/890af16b178545b2b0e3c5894cee5af4, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/cce28a5125274799a9a0c746b4385726, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/bf87499974b1478a97e69c1fa6af8ff9] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp, totalSize=36.7 K 2024-11-25T12:59:50,666 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 890af16b178545b2b0e3c5894cee5af4, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732539586082 2024-11-25T12:59:50,666 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting cce28a5125274799a9a0c746b4385726, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1732539587301 2024-11-25T12:59:50,667 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting bf87499974b1478a97e69c1fa6af8ff9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1732539588037 2024-11-25T12:59:50,672 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6a5867ce2299ba674b3f042fd1fb41c6#C#compaction#433 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:59:50,672 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/913540f313594136af70e9520efb4ed4 is 50, key is test_row_0/C:col10/1732539588072/Put/seqid=0 2024-11-25T12:59:50,719 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/318b803f235c470cbcdd7f83d8003a11 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/318b803f235c470cbcdd7f83d8003a11 2024-11-25T12:59:50,744 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6a5867ce2299ba674b3f042fd1fb41c6/B of 6a5867ce2299ba674b3f042fd1fb41c6 into 318b803f235c470cbcdd7f83d8003a11(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:59:50,744 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6a5867ce2299ba674b3f042fd1fb41c6: 2024-11-25T12:59:50,744 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., storeName=6a5867ce2299ba674b3f042fd1fb41c6/B, priority=13, startTime=1732539590124; duration=0sec 2024-11-25T12:59:50,744 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:59:50,744 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6a5867ce2299ba674b3f042fd1fb41c6:B 2024-11-25T12:59:50,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742337_1513 (size=13085) 2024-11-25T12:59:50,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-25T12:59:50,775 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/913540f313594136af70e9520efb4ed4 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/913540f313594136af70e9520efb4ed4 2024-11-25T12:59:50,780 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6a5867ce2299ba674b3f042fd1fb41c6/C of 6a5867ce2299ba674b3f042fd1fb41c6 into 913540f313594136af70e9520efb4ed4(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:59:50,780 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6a5867ce2299ba674b3f042fd1fb41c6: 2024-11-25T12:59:50,780 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., storeName=6a5867ce2299ba674b3f042fd1fb41c6/C, priority=13, startTime=1732539590124; duration=0sec 2024-11-25T12:59:50,780 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:59:50,780 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6a5867ce2299ba674b3f042fd1fb41c6:C 2024-11-25T12:59:50,801 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/b568fc5612c249d9a8200e786d0547ca 2024-11-25T12:59:50,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/f9ef43217b5e4ff6b628178cd496cbbc as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/f9ef43217b5e4ff6b628178cd496cbbc 2024-11-25T12:59:50,809 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/f9ef43217b5e4ff6b628178cd496cbbc, entries=150, sequenceid=349, filesize=12.0 K 2024-11-25T12:59:50,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/b5076c7407fc4d9bbb9f52b0c05ec7b3 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/b5076c7407fc4d9bbb9f52b0c05ec7b3 2024-11-25T12:59:50,814 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/b5076c7407fc4d9bbb9f52b0c05ec7b3, entries=150, sequenceid=349, filesize=12.0 K 2024-11-25T12:59:50,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/b568fc5612c249d9a8200e786d0547ca as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/b568fc5612c249d9a8200e786d0547ca 2024-11-25T12:59:50,826 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/b568fc5612c249d9a8200e786d0547ca, entries=150, sequenceid=349, filesize=12.0 K 2024-11-25T12:59:50,827 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=0 B/0 for 6a5867ce2299ba674b3f042fd1fb41c6 in 665ms, sequenceid=349, compaction requested=false 2024-11-25T12:59:50,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 6a5867ce2299ba674b3f042fd1fb41c6: 2024-11-25T12:59:50,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:50,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-25T12:59:50,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-25T12:59:50,831 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-25T12:59:50,831 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2130 sec 2024-11-25T12:59:50,835 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 2.2230 sec 2024-11-25T12:59:50,999 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:59:50,999 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6a5867ce2299ba674b3f042fd1fb41c6 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-25T12:59:51,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 6a5867ce2299ba674b3f042fd1fb41c6 2024-11-25T12:59:51,013 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/d57973d1f8454837a15cbe66d9dbf7b0 is 50, key is test_row_0/A:col10/1732539590989/Put/seqid=0 2024-11-25T12:59:51,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742338_1514 (size=9857) 2024-11-25T12:59:51,019 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/d57973d1f8454837a15cbe66d9dbf7b0 2024-11-25T12:59:51,065 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/0bab4e0d495445bb8f03c5082f28e2c3 is 50, key is test_row_0/B:col10/1732539590989/Put/seqid=0 2024-11-25T12:59:51,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742339_1515 (size=9857) 2024-11-25T12:59:51,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:51,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56100 deadline: 1732539651124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:51,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:51,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:51,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:51,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:51,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56082 deadline: 1732539651145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:51,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:51,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:51,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:51,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:51,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56124 deadline: 1732539651146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:51,163 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:51,163 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:51,164 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:51,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:51,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56072 deadline: 1732539651148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:51,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:51,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:51,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:51,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:51,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56108 deadline: 1732539651148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:51,165 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:51,165 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:51,165 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:51,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:51,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56100 deadline: 1732539651259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:51,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:51,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:51,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:51,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:51,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56082 deadline: 1732539651270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:51,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:51,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:51,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:51,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:51,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56072 deadline: 1732539651272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:51,279 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:51,279 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:51,279 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:51,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:51,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56124 deadline: 1732539651272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:51,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:51,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56108 deadline: 1732539651272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:51,279 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:51,280 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:51,280 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:51,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:51,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:51,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:51,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:51,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56100 deadline: 1732539651470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:51,473 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:51,473 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:51,474 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:51,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:51,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56082 deadline: 1732539651488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:51,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:51,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:51,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:51,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:51,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56072 deadline: 1732539651493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:51,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:51,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:51,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:51,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:51,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56124 deadline: 1732539651493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:51,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:51,503 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:51,503 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:51,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56108 deadline: 1732539651493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:51,503 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:51,503 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:51,503 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:51,503 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:51,532 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/0bab4e0d495445bb8f03c5082f28e2c3 2024-11-25T12:59:51,573 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/8d2fd48798854ca787c108dbc4483a15 is 50, key is test_row_0/C:col10/1732539590989/Put/seqid=0 2024-11-25T12:59:51,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742340_1516 (size=9857) 2024-11-25T12:59:51,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:51,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56100 deadline: 1732539651780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:51,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:51,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:51,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:51,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:51,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56082 deadline: 1732539651799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:51,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:51,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:51,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:51,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:51,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56072 deadline: 1732539651811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:51,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:51,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:51,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:51,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:51,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56124 deadline: 1732539651811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:51,815 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:51,815 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:51,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:51,815 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:51,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56108 deadline: 1732539651811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:51,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:51,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:51,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:52,033 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/8d2fd48798854ca787c108dbc4483a15 2024-11-25T12:59:52,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/d57973d1f8454837a15cbe66d9dbf7b0 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/d57973d1f8454837a15cbe66d9dbf7b0 2024-11-25T12:59:52,040 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/d57973d1f8454837a15cbe66d9dbf7b0, entries=100, sequenceid=363, filesize=9.6 K 2024-11-25T12:59:52,040 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/0bab4e0d495445bb8f03c5082f28e2c3 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/0bab4e0d495445bb8f03c5082f28e2c3 2024-11-25T12:59:52,043 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/0bab4e0d495445bb8f03c5082f28e2c3, entries=100, sequenceid=363, filesize=9.6 K 2024-11-25T12:59:52,044 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/8d2fd48798854ca787c108dbc4483a15 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/8d2fd48798854ca787c108dbc4483a15 2024-11-25T12:59:52,047 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/8d2fd48798854ca787c108dbc4483a15, entries=100, sequenceid=363, filesize=9.6 K 2024-11-25T12:59:52,048 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 6a5867ce2299ba674b3f042fd1fb41c6 in 1049ms, sequenceid=363, compaction requested=true 2024-11-25T12:59:52,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6a5867ce2299ba674b3f042fd1fb41c6: 2024-11-25T12:59:52,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6a5867ce2299ba674b3f042fd1fb41c6:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:59:52,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:59:52,048 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:59:52,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6a5867ce2299ba674b3f042fd1fb41c6:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:59:52,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:59:52,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6a5867ce2299ba674b3f042fd1fb41c6:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:59:52,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-25T12:59:52,049 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:59:52,050 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35243 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:59:52,050 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): 6a5867ce2299ba674b3f042fd1fb41c6/A is initiating minor compaction (all files) 2024-11-25T12:59:52,051 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35243 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:59:52,051 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1541): 6a5867ce2299ba674b3f042fd1fb41c6/B is initiating minor compaction (all files) 2024-11-25T12:59:52,051 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6a5867ce2299ba674b3f042fd1fb41c6/B in TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:52,051 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/318b803f235c470cbcdd7f83d8003a11, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/b5076c7407fc4d9bbb9f52b0c05ec7b3, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/0bab4e0d495445bb8f03c5082f28e2c3] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp, totalSize=34.4 K 2024-11-25T12:59:52,050 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6a5867ce2299ba674b3f042fd1fb41c6/A in TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:52,051 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/721ff44d3faf435ca6582e4949379ab3, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/f9ef43217b5e4ff6b628178cd496cbbc, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/d57973d1f8454837a15cbe66d9dbf7b0] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp, totalSize=34.4 K 2024-11-25T12:59:52,051 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 318b803f235c470cbcdd7f83d8003a11, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1732539588037 2024-11-25T12:59:52,051 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5076c7407fc4d9bbb9f52b0c05ec7b3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1732539588765 2024-11-25T12:59:52,051 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 721ff44d3faf435ca6582e4949379ab3, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1732539588037 2024-11-25T12:59:52,052 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting f9ef43217b5e4ff6b628178cd496cbbc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1732539588765 2024-11-25T12:59:52,052 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0bab4e0d495445bb8f03c5082f28e2c3, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1732539590989 2024-11-25T12:59:52,053 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting d57973d1f8454837a15cbe66d9dbf7b0, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1732539590989 2024-11-25T12:59:52,060 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6a5867ce2299ba674b3f042fd1fb41c6#A#compaction#437 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:59:52,061 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/612fa52bc87d44a6b1dbb031d8172cce is 50, key is test_row_0/A:col10/1732539590989/Put/seqid=0 2024-11-25T12:59:52,073 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6a5867ce2299ba674b3f042fd1fb41c6#B#compaction#438 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:59:52,074 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/a2564577c5fd4e86a323c2d40041b9b8 is 50, key is test_row_0/B:col10/1732539590989/Put/seqid=0 2024-11-25T12:59:52,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742341_1517 (size=13187) 2024-11-25T12:59:52,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742342_1518 (size=13187) 2024-11-25T12:59:52,132 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/612fa52bc87d44a6b1dbb031d8172cce as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/612fa52bc87d44a6b1dbb031d8172cce 2024-11-25T12:59:52,139 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/a2564577c5fd4e86a323c2d40041b9b8 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/a2564577c5fd4e86a323c2d40041b9b8 2024-11-25T12:59:52,141 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6a5867ce2299ba674b3f042fd1fb41c6/A of 6a5867ce2299ba674b3f042fd1fb41c6 into 612fa52bc87d44a6b1dbb031d8172cce(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:59:52,141 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6a5867ce2299ba674b3f042fd1fb41c6: 2024-11-25T12:59:52,141 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., storeName=6a5867ce2299ba674b3f042fd1fb41c6/A, priority=13, startTime=1732539592048; duration=0sec 2024-11-25T12:59:52,141 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:59:52,141 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6a5867ce2299ba674b3f042fd1fb41c6:A 2024-11-25T12:59:52,142 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:59:52,144 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35243 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:59:52,145 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): 6a5867ce2299ba674b3f042fd1fb41c6/C is initiating minor compaction (all files) 2024-11-25T12:59:52,145 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6a5867ce2299ba674b3f042fd1fb41c6/C in TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:52,146 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/913540f313594136af70e9520efb4ed4, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/b568fc5612c249d9a8200e786d0547ca, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/8d2fd48798854ca787c108dbc4483a15] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp, totalSize=34.4 K 2024-11-25T12:59:52,147 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 913540f313594136af70e9520efb4ed4, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1732539588037 2024-11-25T12:59:52,148 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting b568fc5612c249d9a8200e786d0547ca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1732539588765 2024-11-25T12:59:52,148 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6a5867ce2299ba674b3f042fd1fb41c6/B of 6a5867ce2299ba674b3f042fd1fb41c6 into a2564577c5fd4e86a323c2d40041b9b8(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:59:52,148 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6a5867ce2299ba674b3f042fd1fb41c6: 2024-11-25T12:59:52,148 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., storeName=6a5867ce2299ba674b3f042fd1fb41c6/B, priority=13, startTime=1732539592048; duration=0sec 2024-11-25T12:59:52,148 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:59:52,148 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6a5867ce2299ba674b3f042fd1fb41c6:B 2024-11-25T12:59:52,148 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d2fd48798854ca787c108dbc4483a15, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1732539590989 2024-11-25T12:59:52,174 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6a5867ce2299ba674b3f042fd1fb41c6#C#compaction#439 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:59:52,174 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/de6ea5b67e4d4f9bafb2c098f1890932 is 50, key is test_row_0/C:col10/1732539590989/Put/seqid=0 2024-11-25T12:59:52,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742343_1519 (size=13187) 2024-11-25T12:59:52,308 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:59:52,308 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6a5867ce2299ba674b3f042fd1fb41c6 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-25T12:59:52,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 6a5867ce2299ba674b3f042fd1fb41c6 2024-11-25T12:59:52,317 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/3e90e12e12334b5c9e56bb9e7d0edfbc is 50, key is test_row_0/A:col10/1732539591145/Put/seqid=0 2024-11-25T12:59:52,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:52,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56082 deadline: 1732539652334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:52,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:52,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56124 deadline: 1732539652335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:52,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:52,343 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:52,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56108 deadline: 1732539652335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:52,343 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:52,343 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:52,346 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:52,346 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:52,346 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:52,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:52,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:52,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:52,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:52,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56100 deadline: 1732539652340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:52,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:52,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56072 deadline: 1732539652341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:52,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:52,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:52,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:52,354 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:52,354 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:52,354 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:52,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742344_1520 (size=12301) 2024-11-25T12:59:52,367 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/3e90e12e12334b5c9e56bb9e7d0edfbc 2024-11-25T12:59:52,384 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/347278e7540244c29fe2c1186e09cdd3 is 50, key is test_row_0/B:col10/1732539591145/Put/seqid=0 2024-11-25T12:59:52,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742345_1521 (size=12301) 2024-11-25T12:59:52,428 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/347278e7540244c29fe2c1186e09cdd3 2024-11-25T12:59:52,447 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/236f3a53f9614a959df4ece24c97bcf0 is 50, key is test_row_0/C:col10/1732539591145/Put/seqid=0 2024-11-25T12:59:52,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:52,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56108 deadline: 1732539652452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:52,459 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:52,459 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:52,459 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:52,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:52,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56124 deadline: 1732539652452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:52,463 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:52,463 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:52,463 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:52,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:52,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56082 deadline: 1732539652452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:52,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:52,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:52,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:52,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:52,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56072 deadline: 1732539652465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:52,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:52,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:52,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:52,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:52,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56100 deadline: 1732539652467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:52,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:52,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:52,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:52,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742346_1522 (size=12301) 2024-11-25T12:59:52,512 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/236f3a53f9614a959df4ece24c97bcf0 2024-11-25T12:59:52,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/3e90e12e12334b5c9e56bb9e7d0edfbc as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/3e90e12e12334b5c9e56bb9e7d0edfbc 2024-11-25T12:59:52,521 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/3e90e12e12334b5c9e56bb9e7d0edfbc, entries=150, sequenceid=392, filesize=12.0 K 2024-11-25T12:59:52,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/347278e7540244c29fe2c1186e09cdd3 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/347278e7540244c29fe2c1186e09cdd3 2024-11-25T12:59:52,529 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/347278e7540244c29fe2c1186e09cdd3, entries=150, sequenceid=392, filesize=12.0 K 2024-11-25T12:59:52,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/236f3a53f9614a959df4ece24c97bcf0 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/236f3a53f9614a959df4ece24c97bcf0 2024-11-25T12:59:52,533 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/236f3a53f9614a959df4ece24c97bcf0, entries=150, sequenceid=392, filesize=12.0 K 2024-11-25T12:59:52,536 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 6a5867ce2299ba674b3f042fd1fb41c6 in 228ms, sequenceid=392, compaction requested=false 2024-11-25T12:59:52,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6a5867ce2299ba674b3f042fd1fb41c6: 2024-11-25T12:59:52,589 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/de6ea5b67e4d4f9bafb2c098f1890932 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/de6ea5b67e4d4f9bafb2c098f1890932 2024-11-25T12:59:52,597 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6a5867ce2299ba674b3f042fd1fb41c6/C of 6a5867ce2299ba674b3f042fd1fb41c6 into de6ea5b67e4d4f9bafb2c098f1890932(size=12.9 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:59:52,597 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6a5867ce2299ba674b3f042fd1fb41c6: 2024-11-25T12:59:52,597 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., storeName=6a5867ce2299ba674b3f042fd1fb41c6/C, priority=13, startTime=1732539592049; duration=0sec 2024-11-25T12:59:52,597 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:59:52,597 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6a5867ce2299ba674b3f042fd1fb41c6:C 2024-11-25T12:59:52,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 6a5867ce2299ba674b3f042fd1fb41c6 2024-11-25T12:59:52,678 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:59:52,678 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6a5867ce2299ba674b3f042fd1fb41c6 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-25T12:59:52,697 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/f82fe5a29d93460785e4e87000cad2ba is 50, key is test_row_0/A:col10/1732539592678/Put/seqid=0 2024-11-25T12:59:52,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742347_1523 (size=19621) 2024-11-25T12:59:52,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:52,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56082 deadline: 1732539652743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:52,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:52,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:52,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:52,755 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:52,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56108 deadline: 1732539652744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:52,755 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:52,755 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:52,755 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:52,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:52,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56072 deadline: 1732539652748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:52,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:52,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56100 deadline: 1732539652750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:52,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:52,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56124 deadline: 1732539652750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:52,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:52,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:52,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:52,763 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:52,763 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:52,763 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:52,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:52,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:52,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:52,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-25T12:59:52,795 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:59:52,800 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:59:52,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-25T12:59:52,809 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:59:52,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-25T12:59:52,810 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:59:52,810 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:59:52,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:52,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56082 deadline: 1732539652860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:52,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:52,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:52,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:52,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:52,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56108 deadline: 1732539652862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:52,868 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:52,868 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:52,868 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:52,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:52,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56124 deadline: 1732539652873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:52,883 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:52,883 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:52,883 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:52,883 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:52,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56072 deadline: 1732539652873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:52,883 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:52,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56100 deadline: 1732539652873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:52,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:52,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:52,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:52,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:52,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:52,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:52,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-25T12:59:52,963 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-25T12:59:52,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:52,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. as already flushing 2024-11-25T12:59:52,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:52,963 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:52,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:52,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:53,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:53,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56082 deadline: 1732539653071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:53,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:53,086 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:53,086 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:53,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56108 deadline: 1732539653071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:53,086 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:53,086 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:53,086 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:53,086 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:53,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:53,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56124 deadline: 1732539653090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:53,101 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:53,101 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:53,101 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:53,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:53,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56072 deadline: 1732539653090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:53,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:53,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:53,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:53,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:53,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56100 deadline: 1732539653091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:53,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:53,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:53,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:53,115 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-25T12:59:53,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:53,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. as already flushing 2024-11-25T12:59:53,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:53,116 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:53,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:53,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:53,128 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/f82fe5a29d93460785e4e87000cad2ba 2024-11-25T12:59:53,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-25T12:59:53,161 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/ddf4d1ad039f4f31ae52aed99d5c6d70 is 50, key is test_row_0/B:col10/1732539592678/Put/seqid=0 2024-11-25T12:59:53,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742348_1524 (size=12301) 2024-11-25T12:59:53,269 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-25T12:59:53,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:53,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. as already flushing 2024-11-25T12:59:53,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:53,269 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:53,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:53,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:53,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:53,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56082 deadline: 1732539653392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:53,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:53,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:53,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:53,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:53,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56108 deadline: 1732539653392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:53,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:53,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:53,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:53,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:53,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56124 deadline: 1732539653411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:53,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:53,418 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:53,418 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:53,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56072 deadline: 1732539653411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:53,418 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:53,419 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:53,419 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:53,419 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:53,424 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-25T12:59:53,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:53,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. as already flushing 2024-11-25T12:59:53,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:53,425 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:53,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:53,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:53,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-25T12:59:53,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:53,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56100 deadline: 1732539653427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:53,448 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:53,448 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:53,448 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:53,578 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-25T12:59:53,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:53,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. as already flushing 2024-11-25T12:59:53,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:53,578 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:53,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:53,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:53,613 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/ddf4d1ad039f4f31ae52aed99d5c6d70 2024-11-25T12:59:53,657 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/bdea42a3e427436aa5b1e38cbcbc0958 is 50, key is test_row_0/C:col10/1732539592678/Put/seqid=0 2024-11-25T12:59:53,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742349_1525 (size=12301) 2024-11-25T12:59:53,737 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-25T12:59:53,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:53,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. as already flushing 2024-11-25T12:59:53,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:53,737 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:53,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:53,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:53,892 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-25T12:59:53,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:53,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. as already flushing 2024-11-25T12:59:53,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:53,892 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:53,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:53,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:53,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:53,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56082 deadline: 1732539653919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:53,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:53,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:53,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:53,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:53,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56108 deadline: 1732539653919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:53,929 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:53,929 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:53,929 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:53,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:53,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56124 deadline: 1732539653930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:53,936 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:53,936 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:53,936 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:53,936 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:53,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56072 deadline: 1732539653930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:53,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:53,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:53,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:53,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-25T12:59:53,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:53,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56100 deadline: 1732539653964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:53,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:53,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:53,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:54,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-25T12:59:54,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:54,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. as already flushing 2024-11-25T12:59:54,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:54,047 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:54,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:54,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:59:54,105 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/bdea42a3e427436aa5b1e38cbcbc0958 2024-11-25T12:59:54,118 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/f82fe5a29d93460785e4e87000cad2ba as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/f82fe5a29d93460785e4e87000cad2ba 2024-11-25T12:59:54,144 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/f82fe5a29d93460785e4e87000cad2ba, entries=300, sequenceid=406, filesize=19.2 K 2024-11-25T12:59:54,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/ddf4d1ad039f4f31ae52aed99d5c6d70 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/ddf4d1ad039f4f31ae52aed99d5c6d70 2024-11-25T12:59:54,161 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/ddf4d1ad039f4f31ae52aed99d5c6d70, entries=150, sequenceid=406, filesize=12.0 K 2024-11-25T12:59:54,162 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/bdea42a3e427436aa5b1e38cbcbc0958 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/bdea42a3e427436aa5b1e38cbcbc0958 2024-11-25T12:59:54,165 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/bdea42a3e427436aa5b1e38cbcbc0958, entries=150, sequenceid=406, filesize=12.0 K 2024-11-25T12:59:54,166 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 6a5867ce2299ba674b3f042fd1fb41c6 in 1488ms, sequenceid=406, compaction requested=true 2024-11-25T12:59:54,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6a5867ce2299ba674b3f042fd1fb41c6: 2024-11-25T12:59:54,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6a5867ce2299ba674b3f042fd1fb41c6:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:59:54,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:59:54,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6a5867ce2299ba674b3f042fd1fb41c6:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:59:54,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-25T12:59:54,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6a5867ce2299ba674b3f042fd1fb41c6:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:59:54,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-25T12:59:54,166 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:59:54,166 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:59:54,167 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 45109 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:59:54,167 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1541): 6a5867ce2299ba674b3f042fd1fb41c6/A is initiating minor compaction (all files) 2024-11-25T12:59:54,167 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6a5867ce2299ba674b3f042fd1fb41c6/A in TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:54,168 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/612fa52bc87d44a6b1dbb031d8172cce, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/3e90e12e12334b5c9e56bb9e7d0edfbc, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/f82fe5a29d93460785e4e87000cad2ba] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp, totalSize=44.1 K 2024-11-25T12:59:54,169 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:59:54,169 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): 6a5867ce2299ba674b3f042fd1fb41c6/C is initiating minor compaction (all files) 2024-11-25T12:59:54,169 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6a5867ce2299ba674b3f042fd1fb41c6/C in TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:54,169 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/de6ea5b67e4d4f9bafb2c098f1890932, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/236f3a53f9614a959df4ece24c97bcf0, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/bdea42a3e427436aa5b1e38cbcbc0958] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp, totalSize=36.9 K 2024-11-25T12:59:54,169 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 612fa52bc87d44a6b1dbb031d8172cce, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1732539588765 2024-11-25T12:59:54,169 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting de6ea5b67e4d4f9bafb2c098f1890932, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1732539588765 2024-11-25T12:59:54,169 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e90e12e12334b5c9e56bb9e7d0edfbc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1732539591144 2024-11-25T12:59:54,169 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 236f3a53f9614a959df4ece24c97bcf0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1732539591144 2024-11-25T12:59:54,169 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting f82fe5a29d93460785e4e87000cad2ba, keycount=300, bloomtype=ROW, size=19.2 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1732539592319 2024-11-25T12:59:54,171 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting bdea42a3e427436aa5b1e38cbcbc0958, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1732539592675 2024-11-25T12:59:54,192 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6a5867ce2299ba674b3f042fd1fb41c6#A#compaction#446 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:59:54,193 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/ff221c1b3a2f434b90dd019822f7331c is 50, key is test_row_0/A:col10/1732539592678/Put/seqid=0 2024-11-25T12:59:54,200 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-25T12:59:54,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:54,204 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 6a5867ce2299ba674b3f042fd1fb41c6 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-25T12:59:54,209 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6a5867ce2299ba674b3f042fd1fb41c6#C#compaction#447 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:59:54,209 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/0b7ed0c46bcd4ac3a7a69c54c2db81ed is 50, key is test_row_0/C:col10/1732539592678/Put/seqid=0 2024-11-25T12:59:54,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/05c2c45b5af143d7b283e4d3104f79c4 is 50, key is test_row_0/A:col10/1732539592729/Put/seqid=0 2024-11-25T12:59:54,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742350_1526 (size=13289) 2024-11-25T12:59:54,273 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/ff221c1b3a2f434b90dd019822f7331c as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/ff221c1b3a2f434b90dd019822f7331c 2024-11-25T12:59:54,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742351_1527 (size=13289) 2024-11-25T12:59:54,284 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6a5867ce2299ba674b3f042fd1fb41c6/A of 6a5867ce2299ba674b3f042fd1fb41c6 into ff221c1b3a2f434b90dd019822f7331c(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:59:54,284 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6a5867ce2299ba674b3f042fd1fb41c6: 2024-11-25T12:59:54,284 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., storeName=6a5867ce2299ba674b3f042fd1fb41c6/A, priority=13, startTime=1732539594166; duration=0sec 2024-11-25T12:59:54,284 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:59:54,284 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6a5867ce2299ba674b3f042fd1fb41c6:A 2024-11-25T12:59:54,284 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:59:54,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742352_1528 (size=12301) 2024-11-25T12:59:54,292 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/0b7ed0c46bcd4ac3a7a69c54c2db81ed as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/0b7ed0c46bcd4ac3a7a69c54c2db81ed 2024-11-25T12:59:54,293 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:59:54,293 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1541): 6a5867ce2299ba674b3f042fd1fb41c6/B is initiating minor compaction (all files) 2024-11-25T12:59:54,293 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6a5867ce2299ba674b3f042fd1fb41c6/B in TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:54,293 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/a2564577c5fd4e86a323c2d40041b9b8, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/347278e7540244c29fe2c1186e09cdd3, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/ddf4d1ad039f4f31ae52aed99d5c6d70] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp, totalSize=36.9 K 2024-11-25T12:59:54,294 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2564577c5fd4e86a323c2d40041b9b8, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1732539588765 2024-11-25T12:59:54,294 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 347278e7540244c29fe2c1186e09cdd3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1732539591144 2024-11-25T12:59:54,295 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting ddf4d1ad039f4f31ae52aed99d5c6d70, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1732539592675 2024-11-25T12:59:54,299 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6a5867ce2299ba674b3f042fd1fb41c6/C of 6a5867ce2299ba674b3f042fd1fb41c6 into 0b7ed0c46bcd4ac3a7a69c54c2db81ed(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:59:54,299 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6a5867ce2299ba674b3f042fd1fb41c6: 2024-11-25T12:59:54,299 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., storeName=6a5867ce2299ba674b3f042fd1fb41c6/C, priority=13, startTime=1732539594166; duration=0sec 2024-11-25T12:59:54,299 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:59:54,299 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6a5867ce2299ba674b3f042fd1fb41c6:C 2024-11-25T12:59:54,309 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6a5867ce2299ba674b3f042fd1fb41c6#B#compaction#449 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:59:54,310 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/9f44b059008545a08f5dc303d89c088e is 50, key is test_row_0/B:col10/1732539592678/Put/seqid=0 2024-11-25T12:59:54,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742353_1529 (size=13289) 2024-11-25T12:59:54,693 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/05c2c45b5af143d7b283e4d3104f79c4 2024-11-25T12:59:54,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/1d20b85a62774984adb8654393d87bd5 is 50, key is test_row_0/B:col10/1732539592729/Put/seqid=0 2024-11-25T12:59:54,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742354_1530 (size=12301) 2024-11-25T12:59:54,780 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/9f44b059008545a08f5dc303d89c088e as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/9f44b059008545a08f5dc303d89c088e 2024-11-25T12:59:54,799 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6a5867ce2299ba674b3f042fd1fb41c6/B of 6a5867ce2299ba674b3f042fd1fb41c6 into 9f44b059008545a08f5dc303d89c088e(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:59:54,799 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6a5867ce2299ba674b3f042fd1fb41c6: 2024-11-25T12:59:54,800 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., storeName=6a5867ce2299ba674b3f042fd1fb41c6/B, priority=13, startTime=1732539594166; duration=0sec 2024-11-25T12:59:54,800 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:59:54,800 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6a5867ce2299ba674b3f042fd1fb41c6:B 2024-11-25T12:59:54,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. as already flushing 2024-11-25T12:59:54,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 6a5867ce2299ba674b3f042fd1fb41c6 2024-11-25T12:59:54,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-25T12:59:54,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:54,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56072 deadline: 1732539654963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:54,972 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:54,972 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:54,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:54,972 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:54,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56108 deadline: 1732539654968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:54,972 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:54,972 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:54,972 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:54,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:54,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56082 deadline: 1732539654969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:54,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:54,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:54,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:54,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:54,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56124 deadline: 1732539654970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:54,983 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:54,983 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:54,983 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:54,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:54,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56100 deadline: 1732539654980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:54,984 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 1ff8009e463e,35905,1732539449246 for 'test_row_0' in 6a5867ce2299ba674b3f042fd1fb41c6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2234 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T12:59:54,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:54,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:54,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:55,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:55,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56108 deadline: 1732539655081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:55,088 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:55,088 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:55,088 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:55,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:55,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56072 deadline: 1732539655087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:55,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:55,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:55,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:55,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:55,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56082 deadline: 1732539655094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:55,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:55,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:55,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:55,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:55,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56124 deadline: 1732539655099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:55,115 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:55,115 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:55,115 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:55,157 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/1d20b85a62774984adb8654393d87bd5 2024-11-25T12:59:55,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/6c22a0a5c726471c862818d20356b4cd is 50, key is test_row_0/C:col10/1732539592729/Put/seqid=0 2024-11-25T12:59:55,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742355_1531 (size=12301) 2024-11-25T12:59:55,172 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/6c22a0a5c726471c862818d20356b4cd 2024-11-25T12:59:55,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/05c2c45b5af143d7b283e4d3104f79c4 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/05c2c45b5af143d7b283e4d3104f79c4 2024-11-25T12:59:55,180 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/05c2c45b5af143d7b283e4d3104f79c4, entries=150, sequenceid=429, filesize=12.0 K 2024-11-25T12:59:55,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/1d20b85a62774984adb8654393d87bd5 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/1d20b85a62774984adb8654393d87bd5 2024-11-25T12:59:55,183 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/1d20b85a62774984adb8654393d87bd5, entries=150, sequenceid=429, filesize=12.0 K 2024-11-25T12:59:55,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/6c22a0a5c726471c862818d20356b4cd as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/6c22a0a5c726471c862818d20356b4cd 2024-11-25T12:59:55,186 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/6c22a0a5c726471c862818d20356b4cd, entries=150, sequenceid=429, filesize=12.0 K 2024-11-25T12:59:55,187 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 6a5867ce2299ba674b3f042fd1fb41c6 in 983ms, sequenceid=429, compaction requested=false 2024-11-25T12:59:55,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 6a5867ce2299ba674b3f042fd1fb41c6: 2024-11-25T12:59:55,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:55,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-25T12:59:55,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-25T12:59:55,190 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-25T12:59:55,190 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3780 sec 2024-11-25T12:59:55,192 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 2.3910 sec 2024-11-25T12:59:55,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 6a5867ce2299ba674b3f042fd1fb41c6 2024-11-25T12:59:55,303 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:59:55,304 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6a5867ce2299ba674b3f042fd1fb41c6 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-25T12:59:55,308 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/a9f72bd5ebe84fbb8416afda5e4181b0 is 50, key is test_row_0/A:col10/1732539595302/Put/seqid=0 2024-11-25T12:59:55,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742356_1532 (size=14741) 2024-11-25T12:59:55,316 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/a9f72bd5ebe84fbb8416afda5e4181b0 2024-11-25T12:59:55,327 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/a94f95fcbe46424fa65305fff17526bf is 50, key is test_row_0/B:col10/1732539595302/Put/seqid=0 2024-11-25T12:59:55,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742357_1533 (size=12301) 2024-11-25T12:59:55,343 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/a94f95fcbe46424fa65305fff17526bf 2024-11-25T12:59:55,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/7c17afa93a5a4d32848e68539054e336 is 50, key is test_row_0/C:col10/1732539595302/Put/seqid=0 2024-11-25T12:59:55,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742358_1534 (size=12301) 2024-11-25T12:59:55,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:55,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56124 deadline: 1732539655362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:55,375 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:55,375 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:55,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:55,375 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:55,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56072 deadline: 1732539655363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:55,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:55,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:55,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:55,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:55,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56082 deadline: 1732539655372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:55,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:55,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:55,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:55,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:55,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56108 deadline: 1732539655375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:55,384 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:55,384 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:55,384 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:55,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:55,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56124 deadline: 1732539655481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:55,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:55,484 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:55,484 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:55,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56072 deadline: 1732539655481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:55,484 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:55,484 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:55,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:55,484 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:55,484 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:55,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56082 deadline: 1732539655481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:55,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:55,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:55,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:55,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:55,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56108 deadline: 1732539655489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:55,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:55,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:55,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:55,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:55,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56124 deadline: 1732539655691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:55,697 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:55,697 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:55,697 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:55,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:55,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56072 deadline: 1732539655692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:55,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:55,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:55,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:55,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:55,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56082 deadline: 1732539655692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:55,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:55,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:55,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:55,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:59:55,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56108 deadline: 1732539655702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:55,709 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 , the old value is region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:55,709 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:59:55,709 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., hostname=1ff8009e463e,35905,1732539449246, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:59:55,748 INFO [Thread-1993 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1993. 2024-11-25T12:59:55,748 DEBUG [Thread-1993 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:59:55,748 DEBUG [Thread-1993 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:59:55,749 DEBUG [Thread-1993 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:59:55,749 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:59:55,758 INFO [Thread-1985 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1985. 2024-11-25T12:59:55,758 DEBUG [Thread-1985 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:59:55,759 DEBUG [Thread-1985 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:59:55,759 DEBUG [Thread-1985 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:59:55,761 INFO [Thread-1989 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1989. 2024-11-25T12:59:55,761 DEBUG [Thread-1989 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:59:55,761 DEBUG [Thread-1989 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:59:55,762 DEBUG [Thread-1989 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:59:55,762 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:59:55,766 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:59:55,773 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/7c17afa93a5a4d32848e68539054e336 2024-11-25T12:59:55,773 INFO [Thread-1987 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1987. 2024-11-25T12:59:55,773 DEBUG [Thread-1987 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:59:55,773 DEBUG [Thread-1987 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:59:55,774 INFO [Thread-1991 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1991. 2024-11-25T12:59:55,774 DEBUG [Thread-1987 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:59:55,774 DEBUG [Thread-1991 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:59:55,774 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:59:55,774 DEBUG [Thread-1991 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:59:55,774 DEBUG [Thread-1991 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:59:55,774 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:59:55,776 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/a9f72bd5ebe84fbb8416afda5e4181b0 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/a9f72bd5ebe84fbb8416afda5e4181b0 2024-11-25T12:59:55,778 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/a9f72bd5ebe84fbb8416afda5e4181b0, entries=200, sequenceid=446, filesize=14.4 K 2024-11-25T12:59:55,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/a94f95fcbe46424fa65305fff17526bf as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/a94f95fcbe46424fa65305fff17526bf 2024-11-25T12:59:55,781 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/a94f95fcbe46424fa65305fff17526bf, entries=150, sequenceid=446, filesize=12.0 K 2024-11-25T12:59:55,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/7c17afa93a5a4d32848e68539054e336 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/7c17afa93a5a4d32848e68539054e336 2024-11-25T12:59:55,784 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/7c17afa93a5a4d32848e68539054e336, entries=150, sequenceid=446, filesize=12.0 K 2024-11-25T12:59:55,784 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 6a5867ce2299ba674b3f042fd1fb41c6 in 481ms, sequenceid=446, compaction requested=true 2024-11-25T12:59:55,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6a5867ce2299ba674b3f042fd1fb41c6: 2024-11-25T12:59:55,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6a5867ce2299ba674b3f042fd1fb41c6:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:59:55,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:59:55,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6a5867ce2299ba674b3f042fd1fb41c6:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:59:55,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:59:55,785 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:59:55,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6a5867ce2299ba674b3f042fd1fb41c6:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:59:55,785 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:59:55,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:59:55,785 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40331 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:59:55,785 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:59:55,785 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1541): 6a5867ce2299ba674b3f042fd1fb41c6/A is initiating minor compaction (all files) 2024-11-25T12:59:55,785 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): 6a5867ce2299ba674b3f042fd1fb41c6/B is initiating minor compaction (all files) 2024-11-25T12:59:55,785 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6a5867ce2299ba674b3f042fd1fb41c6/A in TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:55,785 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6a5867ce2299ba674b3f042fd1fb41c6/B in TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:55,785 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/ff221c1b3a2f434b90dd019822f7331c, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/05c2c45b5af143d7b283e4d3104f79c4, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/a9f72bd5ebe84fbb8416afda5e4181b0] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp, totalSize=39.4 K 2024-11-25T12:59:55,785 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/9f44b059008545a08f5dc303d89c088e, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/1d20b85a62774984adb8654393d87bd5, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/a94f95fcbe46424fa65305fff17526bf] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp, totalSize=37.0 K 2024-11-25T12:59:55,786 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff221c1b3a2f434b90dd019822f7331c, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1732539592675 2024-11-25T12:59:55,786 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f44b059008545a08f5dc303d89c088e, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1732539592675 2024-11-25T12:59:55,786 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05c2c45b5af143d7b283e4d3104f79c4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1732539592729 2024-11-25T12:59:55,786 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d20b85a62774984adb8654393d87bd5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1732539592729 2024-11-25T12:59:55,786 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9f72bd5ebe84fbb8416afda5e4181b0, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1732539594957 2024-11-25T12:59:55,786 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting a94f95fcbe46424fa65305fff17526bf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1732539594957 2024-11-25T12:59:55,791 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6a5867ce2299ba674b3f042fd1fb41c6#B#compaction#455 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:59:55,791 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6a5867ce2299ba674b3f042fd1fb41c6#A#compaction#456 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:59:55,791 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/10c45f48837449e0b8bba90039e9dbe8 is 50, key is test_row_0/B:col10/1732539595302/Put/seqid=0 2024-11-25T12:59:55,791 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/c496d0305aa448f185149e528acc7960 is 50, key is test_row_0/A:col10/1732539595302/Put/seqid=0 2024-11-25T12:59:55,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742359_1535 (size=13391) 2024-11-25T12:59:55,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742360_1536 (size=13391) 2024-11-25T12:59:55,801 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/10c45f48837449e0b8bba90039e9dbe8 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/10c45f48837449e0b8bba90039e9dbe8 2024-11-25T12:59:55,806 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6a5867ce2299ba674b3f042fd1fb41c6/B of 6a5867ce2299ba674b3f042fd1fb41c6 into 10c45f48837449e0b8bba90039e9dbe8(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:59:55,806 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6a5867ce2299ba674b3f042fd1fb41c6: 2024-11-25T12:59:55,806 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., storeName=6a5867ce2299ba674b3f042fd1fb41c6/B, priority=13, startTime=1732539595785; duration=0sec 2024-11-25T12:59:55,806 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:59:55,806 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6a5867ce2299ba674b3f042fd1fb41c6:B 2024-11-25T12:59:55,806 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:59:55,807 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:59:55,807 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): 6a5867ce2299ba674b3f042fd1fb41c6/C is initiating minor compaction (all files) 2024-11-25T12:59:55,807 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6a5867ce2299ba674b3f042fd1fb41c6/C in TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:55,807 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/0b7ed0c46bcd4ac3a7a69c54c2db81ed, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/6c22a0a5c726471c862818d20356b4cd, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/7c17afa93a5a4d32848e68539054e336] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp, totalSize=37.0 K 2024-11-25T12:59:55,807 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b7ed0c46bcd4ac3a7a69c54c2db81ed, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1732539592675 2024-11-25T12:59:55,810 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c22a0a5c726471c862818d20356b4cd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1732539592729 2024-11-25T12:59:55,810 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c17afa93a5a4d32848e68539054e336, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1732539594957 2024-11-25T12:59:55,821 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6a5867ce2299ba674b3f042fd1fb41c6#C#compaction#457 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:59:55,822 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/fc1d28cc5fc246cea052b594c3732c28 is 50, key is test_row_0/C:col10/1732539595302/Put/seqid=0 2024-11-25T12:59:55,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742361_1537 (size=13391) 2024-11-25T12:59:56,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 6a5867ce2299ba674b3f042fd1fb41c6 2024-11-25T12:59:56,012 INFO [Thread-1974 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1974. 2024-11-25T12:59:56,012 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:59:56,012 DEBUG [Thread-1974 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:59:56,012 DEBUG [Thread-1974 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:59:56,012 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6a5867ce2299ba674b3f042fd1fb41c6 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-25T12:59:56,012 DEBUG [Thread-1974 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:59:56,012 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:59:56,013 INFO [Thread-1976 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1976. 2024-11-25T12:59:56,013 INFO [Thread-1982 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1982. 2024-11-25T12:59:56,013 DEBUG [Thread-1982 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:59:56,013 DEBUG [Thread-1976 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:59:56,013 DEBUG [Thread-1976 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:59:56,013 DEBUG [Thread-1982 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:59:56,013 DEBUG [Thread-1976 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:59:56,013 DEBUG [Thread-1982 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:59:56,015 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:59:56,015 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:59:56,026 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/3f1ca8c3fb2649d8aeb7adce85f22aee is 50, key is test_row_0/A:col10/1732539596011/Put/seqid=0 2024-11-25T12:59:56,028 INFO [Thread-1980 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1980. 2024-11-25T12:59:56,028 DEBUG [Thread-1980 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:59:56,028 DEBUG [Thread-1980 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:59:56,028 DEBUG [Thread-1980 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:59:56,028 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:59:56,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742362_1538 (size=12301) 2024-11-25T12:59:56,040 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=472 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/3f1ca8c3fb2649d8aeb7adce85f22aee 2024-11-25T12:59:56,048 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/41343c73340847dd8cdbfad20aa3b026 is 50, key is test_row_0/B:col10/1732539596011/Put/seqid=0 2024-11-25T12:59:56,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742363_1539 (size=12301) 2024-11-25T12:59:56,060 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=472 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/41343c73340847dd8cdbfad20aa3b026 2024-11-25T12:59:56,073 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/e5bd2523eb184107a53db1c73772d4c8 is 50, key is test_row_0/C:col10/1732539596011/Put/seqid=0 2024-11-25T12:59:56,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742364_1540 (size=12301) 2024-11-25T12:59:56,203 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/c496d0305aa448f185149e528acc7960 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/c496d0305aa448f185149e528acc7960 2024-11-25T12:59:56,214 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6a5867ce2299ba674b3f042fd1fb41c6/A of 6a5867ce2299ba674b3f042fd1fb41c6 into c496d0305aa448f185149e528acc7960(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:59:56,214 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6a5867ce2299ba674b3f042fd1fb41c6: 2024-11-25T12:59:56,214 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., storeName=6a5867ce2299ba674b3f042fd1fb41c6/A, priority=13, startTime=1732539595784; duration=0sec 2024-11-25T12:59:56,214 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:59:56,214 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6a5867ce2299ba674b3f042fd1fb41c6:A 2024-11-25T12:59:56,238 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/fc1d28cc5fc246cea052b594c3732c28 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/fc1d28cc5fc246cea052b594c3732c28 2024-11-25T12:59:56,245 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6a5867ce2299ba674b3f042fd1fb41c6/C of 6a5867ce2299ba674b3f042fd1fb41c6 into fc1d28cc5fc246cea052b594c3732c28(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:59:56,245 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6a5867ce2299ba674b3f042fd1fb41c6: 2024-11-25T12:59:56,245 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., storeName=6a5867ce2299ba674b3f042fd1fb41c6/C, priority=13, startTime=1732539595785; duration=0sec 2024-11-25T12:59:56,245 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:59:56,245 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6a5867ce2299ba674b3f042fd1fb41c6:C 2024-11-25T12:59:56,489 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=472 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/e5bd2523eb184107a53db1c73772d4c8 2024-11-25T12:59:56,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/3f1ca8c3fb2649d8aeb7adce85f22aee as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/3f1ca8c3fb2649d8aeb7adce85f22aee 2024-11-25T12:59:56,516 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/3f1ca8c3fb2649d8aeb7adce85f22aee, entries=150, sequenceid=472, filesize=12.0 K 2024-11-25T12:59:56,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/41343c73340847dd8cdbfad20aa3b026 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/41343c73340847dd8cdbfad20aa3b026 2024-11-25T12:59:56,524 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/41343c73340847dd8cdbfad20aa3b026, entries=150, sequenceid=472, filesize=12.0 K 2024-11-25T12:59:56,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/e5bd2523eb184107a53db1c73772d4c8 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/e5bd2523eb184107a53db1c73772d4c8 2024-11-25T12:59:56,537 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/e5bd2523eb184107a53db1c73772d4c8, entries=150, sequenceid=472, filesize=12.0 K 2024-11-25T12:59:56,538 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=6.71 KB/6870 for 6a5867ce2299ba674b3f042fd1fb41c6 in 526ms, sequenceid=472, compaction requested=false 2024-11-25T12:59:56,538 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6a5867ce2299ba674b3f042fd1fb41c6: 2024-11-25T12:59:56,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-25T12:59:56,971 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:59:57,011 INFO [Thread-1978 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1978. 2024-11-25T12:59:57,011 DEBUG [Thread-1978 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:59:57,011 DEBUG [Thread-1978 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:59:57,011 DEBUG [Thread-1978 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:59:57,011 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:59:57,011 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-25T12:59:57,012 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 75 2024-11-25T12:59:57,012 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 82 2024-11-25T12:59:57,012 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 70 2024-11-25T12:59:57,012 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 74 2024-11-25T12:59:57,012 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 72 2024-11-25T12:59:57,012 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-25T12:59:57,012 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-25T12:59:57,012 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 983 2024-11-25T12:59:57,012 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2949 rows 2024-11-25T12:59:57,012 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 990 2024-11-25T12:59:57,012 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2945 rows 2024-11-25T12:59:57,012 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 989 2024-11-25T12:59:57,012 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2967 rows 2024-11-25T12:59:57,012 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 993 2024-11-25T12:59:57,012 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2978 rows 2024-11-25T12:59:57,012 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 981 2024-11-25T12:59:57,012 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2943 rows 2024-11-25T12:59:57,012 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T12:59:57,012 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T12:59:57,012 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:59:57,012 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:59:57,012 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:59:57,013 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-25T12:59:57,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-25T12:59:57,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-25T12:59:57,015 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732539597015"}]},"ts":"1732539597015"} 2024-11-25T12:59:57,016 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-25T12:59:57,016 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-25T12:59:57,017 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-25T12:59:57,018 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6a5867ce2299ba674b3f042fd1fb41c6, UNASSIGN}] 2024-11-25T12:59:57,019 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6a5867ce2299ba674b3f042fd1fb41c6, UNASSIGN 2024-11-25T12:59:57,019 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=129 updating hbase:meta row=6a5867ce2299ba674b3f042fd1fb41c6, regionState=CLOSING, regionLocation=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:57,020 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6a5867ce2299ba674b3f042fd1fb41c6, UNASSIGN because future has completed 2024-11-25T12:59:57,020 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-25T12:59:57,020 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246}] 2024-11-25T12:59:57,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-25T12:59:57,173 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] handler.UnassignRegionHandler(122): Close 6a5867ce2299ba674b3f042fd1fb41c6 2024-11-25T12:59:57,173 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-25T12:59:57,173 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1722): Closing 6a5867ce2299ba674b3f042fd1fb41c6, disabling compactions & flushes 2024-11-25T12:59:57,173 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:57,173 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:57,173 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. after waiting 0 ms 2024-11-25T12:59:57,173 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:57,173 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(2902): Flushing 6a5867ce2299ba674b3f042fd1fb41c6 3/3 column families, dataSize=13.42 KB heapSize=35.91 KB 2024-11-25T12:59:57,177 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/5973352d5fd9409d87b319e83a8fa2ef is 50, key is test_row_0/A:col10/1732539597010/Put/seqid=0 2024-11-25T12:59:57,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742365_1541 (size=7415) 2024-11-25T12:59:57,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-25T12:59:57,355 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-25T12:59:57,580 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/5973352d5fd9409d87b319e83a8fa2ef 2024-11-25T12:59:57,586 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/4e54d9acaa8848a894174a6fbd57c3f9 is 50, key is test_row_0/B:col10/1732539597010/Put/seqid=0 2024-11-25T12:59:57,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742366_1542 (size=7415) 2024-11-25T12:59:57,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-25T12:59:57,989 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/4e54d9acaa8848a894174a6fbd57c3f9 2024-11-25T12:59:57,994 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/fef5db707cc246158afde0c7c0020b47 is 50, key is test_row_0/C:col10/1732539597010/Put/seqid=0 2024-11-25T12:59:58,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742367_1543 (size=7415) 2024-11-25T12:59:58,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-25T12:59:58,401 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/fef5db707cc246158afde0c7c0020b47 2024-11-25T12:59:58,404 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/A/5973352d5fd9409d87b319e83a8fa2ef as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/5973352d5fd9409d87b319e83a8fa2ef 2024-11-25T12:59:58,407 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/5973352d5fd9409d87b319e83a8fa2ef, entries=50, sequenceid=479, filesize=7.2 K 2024-11-25T12:59:58,407 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/B/4e54d9acaa8848a894174a6fbd57c3f9 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/4e54d9acaa8848a894174a6fbd57c3f9 2024-11-25T12:59:58,409 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/4e54d9acaa8848a894174a6fbd57c3f9, entries=50, sequenceid=479, filesize=7.2 K 2024-11-25T12:59:58,410 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/.tmp/C/fef5db707cc246158afde0c7c0020b47 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/fef5db707cc246158afde0c7c0020b47 2024-11-25T12:59:58,413 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/fef5db707cc246158afde0c7c0020b47, entries=50, sequenceid=479, filesize=7.2 K 2024-11-25T12:59:58,413 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for 6a5867ce2299ba674b3f042fd1fb41c6 in 1240ms, sequenceid=479, compaction requested=true 2024-11-25T12:59:58,414 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/bce81b1adf2f4b8a8c6f849811591ea8, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/4bcb91af5f244b5c9eefcc1e99dd978c, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/90fcc36a71984e4597da967c8d21e35e, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/97e8728d9ac142aa8727fe3ff69e4e27, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/6c724c6f1bad493289b80feaa8273e21, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/6b19b9811a2a46f8abb7344a29b3336e, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/cf48b19c7a8a4d7ebda03d3f11056d47, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/7cabbe60ed29439291d864fd27ec933e, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/35ba11c833b64979810e0ae19386de1a, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/72441409117d4bbfa349b45ccb6a07a3, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/a5b2a413b848452d98a761dc104f29ed, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/d11f5e665b914943b0acffa4f1be33fd, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/8f2c51c193dc4b0b87f1f5cf2c9d28a4, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/1788c4e191bb45bfbb999d0358791ab1, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/3983032290e34afc8aeb0bb748ebe92a, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/001f5af38ca24e0985f10fe419451bfb, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/fbcb82c00bb04e51b1f72f2f1d0e66ff, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/851c4b92b9bb48dc960affd9f61f174c, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/15a204ce13254ac78a9846a6ef044d83, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/bb5609380cd04ed482d5e9b1e1ab751a, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/5075ebac0a99454d8f2f7e3fdbf73a0e, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/0cffcfd00b5f494b857dcf50e7156116, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/547507aa9ea54972b3339666e82d7de2, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/721ff44d3faf435ca6582e4949379ab3, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/f9ef43217b5e4ff6b628178cd496cbbc, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/612fa52bc87d44a6b1dbb031d8172cce, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/d57973d1f8454837a15cbe66d9dbf7b0, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/3e90e12e12334b5c9e56bb9e7d0edfbc, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/f82fe5a29d93460785e4e87000cad2ba, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/ff221c1b3a2f434b90dd019822f7331c, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/05c2c45b5af143d7b283e4d3104f79c4, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/a9f72bd5ebe84fbb8416afda5e4181b0] to archive 2024-11-25T12:59:58,414 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T12:59:58,415 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/bce81b1adf2f4b8a8c6f849811591ea8 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/bce81b1adf2f4b8a8c6f849811591ea8 2024-11-25T12:59:58,416 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/4bcb91af5f244b5c9eefcc1e99dd978c to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/4bcb91af5f244b5c9eefcc1e99dd978c 2024-11-25T12:59:58,417 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/90fcc36a71984e4597da967c8d21e35e to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/90fcc36a71984e4597da967c8d21e35e 2024-11-25T12:59:58,417 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/97e8728d9ac142aa8727fe3ff69e4e27 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/97e8728d9ac142aa8727fe3ff69e4e27 2024-11-25T12:59:58,418 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/6c724c6f1bad493289b80feaa8273e21 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/6c724c6f1bad493289b80feaa8273e21 2024-11-25T12:59:58,419 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/6b19b9811a2a46f8abb7344a29b3336e to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/6b19b9811a2a46f8abb7344a29b3336e 2024-11-25T12:59:58,420 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/cf48b19c7a8a4d7ebda03d3f11056d47 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/cf48b19c7a8a4d7ebda03d3f11056d47 2024-11-25T12:59:58,420 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/7cabbe60ed29439291d864fd27ec933e to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/7cabbe60ed29439291d864fd27ec933e 2024-11-25T12:59:58,421 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/35ba11c833b64979810e0ae19386de1a to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/35ba11c833b64979810e0ae19386de1a 2024-11-25T12:59:58,422 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/72441409117d4bbfa349b45ccb6a07a3 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/72441409117d4bbfa349b45ccb6a07a3 2024-11-25T12:59:58,423 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/a5b2a413b848452d98a761dc104f29ed to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/a5b2a413b848452d98a761dc104f29ed 2024-11-25T12:59:58,423 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/d11f5e665b914943b0acffa4f1be33fd to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/d11f5e665b914943b0acffa4f1be33fd 2024-11-25T12:59:58,424 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/8f2c51c193dc4b0b87f1f5cf2c9d28a4 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/8f2c51c193dc4b0b87f1f5cf2c9d28a4 2024-11-25T12:59:58,425 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/1788c4e191bb45bfbb999d0358791ab1 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/1788c4e191bb45bfbb999d0358791ab1 2024-11-25T12:59:58,425 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/3983032290e34afc8aeb0bb748ebe92a to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/3983032290e34afc8aeb0bb748ebe92a 2024-11-25T12:59:58,426 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/001f5af38ca24e0985f10fe419451bfb to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/001f5af38ca24e0985f10fe419451bfb 2024-11-25T12:59:58,426 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/fbcb82c00bb04e51b1f72f2f1d0e66ff to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/fbcb82c00bb04e51b1f72f2f1d0e66ff 2024-11-25T12:59:58,427 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/851c4b92b9bb48dc960affd9f61f174c to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/851c4b92b9bb48dc960affd9f61f174c 2024-11-25T12:59:58,428 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/15a204ce13254ac78a9846a6ef044d83 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/15a204ce13254ac78a9846a6ef044d83 2024-11-25T12:59:58,428 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/bb5609380cd04ed482d5e9b1e1ab751a to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/bb5609380cd04ed482d5e9b1e1ab751a 2024-11-25T12:59:58,429 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/5075ebac0a99454d8f2f7e3fdbf73a0e to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/5075ebac0a99454d8f2f7e3fdbf73a0e 2024-11-25T12:59:58,429 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/0cffcfd00b5f494b857dcf50e7156116 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/0cffcfd00b5f494b857dcf50e7156116 2024-11-25T12:59:58,430 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/547507aa9ea54972b3339666e82d7de2 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/547507aa9ea54972b3339666e82d7de2 2024-11-25T12:59:58,431 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/721ff44d3faf435ca6582e4949379ab3 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/721ff44d3faf435ca6582e4949379ab3 2024-11-25T12:59:58,432 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/f9ef43217b5e4ff6b628178cd496cbbc to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/f9ef43217b5e4ff6b628178cd496cbbc 2024-11-25T12:59:58,432 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/612fa52bc87d44a6b1dbb031d8172cce to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/612fa52bc87d44a6b1dbb031d8172cce 2024-11-25T12:59:58,433 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/d57973d1f8454837a15cbe66d9dbf7b0 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/d57973d1f8454837a15cbe66d9dbf7b0 2024-11-25T12:59:58,434 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/3e90e12e12334b5c9e56bb9e7d0edfbc to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/3e90e12e12334b5c9e56bb9e7d0edfbc 2024-11-25T12:59:58,435 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/f82fe5a29d93460785e4e87000cad2ba to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/f82fe5a29d93460785e4e87000cad2ba 2024-11-25T12:59:58,436 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/ff221c1b3a2f434b90dd019822f7331c to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/ff221c1b3a2f434b90dd019822f7331c 2024-11-25T12:59:58,436 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/05c2c45b5af143d7b283e4d3104f79c4 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/05c2c45b5af143d7b283e4d3104f79c4 2024-11-25T12:59:58,437 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/a9f72bd5ebe84fbb8416afda5e4181b0 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/a9f72bd5ebe84fbb8416afda5e4181b0 2024-11-25T12:59:58,439 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/a7c6c10ee9bc41e9bc0380388941995a, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/36376b0bc10c42d780ec9daa3f2bc865, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/9ddd08755bd54490aa83720c6aecfb85, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/9ea8e2cd914a4ed0b5582d5802fcc928, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/acaf5b1589d8495696880c05a36e4860, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/01780359b71342cc81992d65a78b358d, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/69b147cb40c14cf899e8348053abaf9a, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/8904b306030d42ce903606ba9f8be7b9, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/38820115db0840338ee44e56420aafd7, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/d0e431729c2f47daaf371948849e2f93, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/1bd17b8424c14cdd91cbe8baf004ce2b, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/c7c485d518714f4cad1de6e4a6ec35f4, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/87c623cd79a342f7bd6acb4f73d6c004, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/98acac1c2461476d8ae643667c3dfc8d, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/b95427c220764406bf3b4a2db4e9c46e, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/e75c17021add424486992224f943be3f, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/1776e69b72b647b696953594bde4294e, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/99ec25698a104627862856ef539b1565, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/20acd0ae2e5940c883fba12c16b5bfd5, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/3c04192c654d4b20a34072f3d52f592c, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/b3ef34258c3d42cfbe033d41b764de02, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/ddbd42152a114665b1f4f90479e22376, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/318b803f235c470cbcdd7f83d8003a11, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/70107ca6483747f498c66a4bced9f6b1, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/b5076c7407fc4d9bbb9f52b0c05ec7b3, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/a2564577c5fd4e86a323c2d40041b9b8, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/0bab4e0d495445bb8f03c5082f28e2c3, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/347278e7540244c29fe2c1186e09cdd3, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/9f44b059008545a08f5dc303d89c088e, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/ddf4d1ad039f4f31ae52aed99d5c6d70, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/1d20b85a62774984adb8654393d87bd5, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/a94f95fcbe46424fa65305fff17526bf] to archive 2024-11-25T12:59:58,440 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T12:59:58,441 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/a7c6c10ee9bc41e9bc0380388941995a to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/a7c6c10ee9bc41e9bc0380388941995a 2024-11-25T12:59:58,442 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/36376b0bc10c42d780ec9daa3f2bc865 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/36376b0bc10c42d780ec9daa3f2bc865 2024-11-25T12:59:58,443 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/9ddd08755bd54490aa83720c6aecfb85 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/9ddd08755bd54490aa83720c6aecfb85 2024-11-25T12:59:58,443 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/9ea8e2cd914a4ed0b5582d5802fcc928 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/9ea8e2cd914a4ed0b5582d5802fcc928 2024-11-25T12:59:58,444 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/acaf5b1589d8495696880c05a36e4860 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/acaf5b1589d8495696880c05a36e4860 2024-11-25T12:59:58,445 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/01780359b71342cc81992d65a78b358d to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/01780359b71342cc81992d65a78b358d 2024-11-25T12:59:58,446 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/69b147cb40c14cf899e8348053abaf9a to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/69b147cb40c14cf899e8348053abaf9a 2024-11-25T12:59:58,447 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/8904b306030d42ce903606ba9f8be7b9 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/8904b306030d42ce903606ba9f8be7b9 2024-11-25T12:59:58,448 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/38820115db0840338ee44e56420aafd7 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/38820115db0840338ee44e56420aafd7 2024-11-25T12:59:58,448 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/d0e431729c2f47daaf371948849e2f93 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/d0e431729c2f47daaf371948849e2f93 2024-11-25T12:59:58,449 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/1bd17b8424c14cdd91cbe8baf004ce2b to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/1bd17b8424c14cdd91cbe8baf004ce2b 2024-11-25T12:59:58,450 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/c7c485d518714f4cad1de6e4a6ec35f4 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/c7c485d518714f4cad1de6e4a6ec35f4 2024-11-25T12:59:58,450 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/87c623cd79a342f7bd6acb4f73d6c004 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/87c623cd79a342f7bd6acb4f73d6c004 2024-11-25T12:59:58,451 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/98acac1c2461476d8ae643667c3dfc8d to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/98acac1c2461476d8ae643667c3dfc8d 2024-11-25T12:59:58,452 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/b95427c220764406bf3b4a2db4e9c46e to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/b95427c220764406bf3b4a2db4e9c46e 2024-11-25T12:59:58,453 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/e75c17021add424486992224f943be3f to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/e75c17021add424486992224f943be3f 2024-11-25T12:59:58,453 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/1776e69b72b647b696953594bde4294e to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/1776e69b72b647b696953594bde4294e 2024-11-25T12:59:58,454 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/99ec25698a104627862856ef539b1565 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/99ec25698a104627862856ef539b1565 2024-11-25T12:59:58,455 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/20acd0ae2e5940c883fba12c16b5bfd5 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/20acd0ae2e5940c883fba12c16b5bfd5 2024-11-25T12:59:58,455 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/3c04192c654d4b20a34072f3d52f592c to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/3c04192c654d4b20a34072f3d52f592c 2024-11-25T12:59:58,456 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/b3ef34258c3d42cfbe033d41b764de02 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/b3ef34258c3d42cfbe033d41b764de02 2024-11-25T12:59:58,457 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/ddbd42152a114665b1f4f90479e22376 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/ddbd42152a114665b1f4f90479e22376 2024-11-25T12:59:58,458 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/318b803f235c470cbcdd7f83d8003a11 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/318b803f235c470cbcdd7f83d8003a11 2024-11-25T12:59:58,458 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/70107ca6483747f498c66a4bced9f6b1 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/70107ca6483747f498c66a4bced9f6b1 2024-11-25T12:59:58,459 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/b5076c7407fc4d9bbb9f52b0c05ec7b3 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/b5076c7407fc4d9bbb9f52b0c05ec7b3 2024-11-25T12:59:58,460 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/a2564577c5fd4e86a323c2d40041b9b8 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/a2564577c5fd4e86a323c2d40041b9b8 2024-11-25T12:59:58,461 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/0bab4e0d495445bb8f03c5082f28e2c3 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/0bab4e0d495445bb8f03c5082f28e2c3 2024-11-25T12:59:58,462 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/347278e7540244c29fe2c1186e09cdd3 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/347278e7540244c29fe2c1186e09cdd3 2024-11-25T12:59:58,462 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/9f44b059008545a08f5dc303d89c088e to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/9f44b059008545a08f5dc303d89c088e 2024-11-25T12:59:58,463 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/ddf4d1ad039f4f31ae52aed99d5c6d70 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/ddf4d1ad039f4f31ae52aed99d5c6d70 2024-11-25T12:59:58,464 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/1d20b85a62774984adb8654393d87bd5 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/1d20b85a62774984adb8654393d87bd5 2024-11-25T12:59:58,465 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/a94f95fcbe46424fa65305fff17526bf to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/a94f95fcbe46424fa65305fff17526bf 2024-11-25T12:59:58,466 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/faae89c614b2438cb4b1d26073397ae6, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/0be2bedc113d4fb5a9866cab074458df, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/965fa3de64834ea089094935e82579d5, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/6eb0620f29654f479eb5b07e620013f1, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/8a91b22160264d5bb578957e259377a3, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/8255c7cce7ce4b87abea7212fa804499, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/5186f903f012462788987471f8f8c839, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/bcc6ad99feb24b56a76a919d0c414713, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/26382852102148f4a936c279df67a892, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/05c97d1de0f44baa83dd80d54ae9dcc5, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/15447463641a45b491208e08f896cae6, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/483313348eab4a8e9ff723d02a9dcead, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/b7f61d617cb94bf38de9aadcd548f2f4, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/b09eceb035c1467b8870b9d89f8b0929, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/d867629f5f3f4f2ca43e483ae05325ee, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/c9130b086016474f972e01930e5fd492, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/8932749270bc4fd38aaafad6076f8881, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/bafa155d6c8148239c44b6b4d7565f1f, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/2d68fd4b7d9c46afbc7a2cc324a5f9d2, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/890af16b178545b2b0e3c5894cee5af4, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/618b4b25ff944c31a50eaa50f64877ee, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/cce28a5125274799a9a0c746b4385726, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/913540f313594136af70e9520efb4ed4, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/bf87499974b1478a97e69c1fa6af8ff9, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/b568fc5612c249d9a8200e786d0547ca, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/de6ea5b67e4d4f9bafb2c098f1890932, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/8d2fd48798854ca787c108dbc4483a15, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/236f3a53f9614a959df4ece24c97bcf0, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/0b7ed0c46bcd4ac3a7a69c54c2db81ed, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/bdea42a3e427436aa5b1e38cbcbc0958, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/6c22a0a5c726471c862818d20356b4cd, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/7c17afa93a5a4d32848e68539054e336] to archive 2024-11-25T12:59:58,467 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T12:59:58,468 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/faae89c614b2438cb4b1d26073397ae6 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/faae89c614b2438cb4b1d26073397ae6 2024-11-25T12:59:58,469 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/0be2bedc113d4fb5a9866cab074458df to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/0be2bedc113d4fb5a9866cab074458df 2024-11-25T12:59:58,470 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/965fa3de64834ea089094935e82579d5 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/965fa3de64834ea089094935e82579d5 2024-11-25T12:59:58,470 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/6eb0620f29654f479eb5b07e620013f1 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/6eb0620f29654f479eb5b07e620013f1 2024-11-25T12:59:58,471 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/8a91b22160264d5bb578957e259377a3 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/8a91b22160264d5bb578957e259377a3 2024-11-25T12:59:58,472 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/8255c7cce7ce4b87abea7212fa804499 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/8255c7cce7ce4b87abea7212fa804499 2024-11-25T12:59:58,473 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/5186f903f012462788987471f8f8c839 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/5186f903f012462788987471f8f8c839 2024-11-25T12:59:58,474 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/bcc6ad99feb24b56a76a919d0c414713 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/bcc6ad99feb24b56a76a919d0c414713 2024-11-25T12:59:58,474 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/26382852102148f4a936c279df67a892 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/26382852102148f4a936c279df67a892 2024-11-25T12:59:58,475 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/05c97d1de0f44baa83dd80d54ae9dcc5 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/05c97d1de0f44baa83dd80d54ae9dcc5 2024-11-25T12:59:58,476 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/15447463641a45b491208e08f896cae6 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/15447463641a45b491208e08f896cae6 2024-11-25T12:59:58,477 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/483313348eab4a8e9ff723d02a9dcead to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/483313348eab4a8e9ff723d02a9dcead 2024-11-25T12:59:58,478 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/b7f61d617cb94bf38de9aadcd548f2f4 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/b7f61d617cb94bf38de9aadcd548f2f4 2024-11-25T12:59:58,478 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/b09eceb035c1467b8870b9d89f8b0929 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/b09eceb035c1467b8870b9d89f8b0929 2024-11-25T12:59:58,479 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/d867629f5f3f4f2ca43e483ae05325ee to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/d867629f5f3f4f2ca43e483ae05325ee 2024-11-25T12:59:58,480 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/c9130b086016474f972e01930e5fd492 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/c9130b086016474f972e01930e5fd492 2024-11-25T12:59:58,481 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/8932749270bc4fd38aaafad6076f8881 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/8932749270bc4fd38aaafad6076f8881 2024-11-25T12:59:58,481 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/bafa155d6c8148239c44b6b4d7565f1f to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/bafa155d6c8148239c44b6b4d7565f1f 2024-11-25T12:59:58,491 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/2d68fd4b7d9c46afbc7a2cc324a5f9d2 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/2d68fd4b7d9c46afbc7a2cc324a5f9d2 2024-11-25T12:59:58,492 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/890af16b178545b2b0e3c5894cee5af4 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/890af16b178545b2b0e3c5894cee5af4 2024-11-25T12:59:58,493 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/618b4b25ff944c31a50eaa50f64877ee to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/618b4b25ff944c31a50eaa50f64877ee 2024-11-25T12:59:58,495 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/cce28a5125274799a9a0c746b4385726 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/cce28a5125274799a9a0c746b4385726 2024-11-25T12:59:58,497 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/913540f313594136af70e9520efb4ed4 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/913540f313594136af70e9520efb4ed4 2024-11-25T12:59:58,498 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/bf87499974b1478a97e69c1fa6af8ff9 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/bf87499974b1478a97e69c1fa6af8ff9 2024-11-25T12:59:58,499 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/b568fc5612c249d9a8200e786d0547ca to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/b568fc5612c249d9a8200e786d0547ca 2024-11-25T12:59:58,500 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/de6ea5b67e4d4f9bafb2c098f1890932 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/de6ea5b67e4d4f9bafb2c098f1890932 2024-11-25T12:59:58,501 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/8d2fd48798854ca787c108dbc4483a15 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/8d2fd48798854ca787c108dbc4483a15 2024-11-25T12:59:58,501 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/236f3a53f9614a959df4ece24c97bcf0 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/236f3a53f9614a959df4ece24c97bcf0 2024-11-25T12:59:58,502 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/0b7ed0c46bcd4ac3a7a69c54c2db81ed to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/0b7ed0c46bcd4ac3a7a69c54c2db81ed 2024-11-25T12:59:58,503 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/bdea42a3e427436aa5b1e38cbcbc0958 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/bdea42a3e427436aa5b1e38cbcbc0958 2024-11-25T12:59:58,504 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/6c22a0a5c726471c862818d20356b4cd to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/6c22a0a5c726471c862818d20356b4cd 2024-11-25T12:59:58,504 DEBUG [StoreCloser-TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/7c17afa93a5a4d32848e68539054e336 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/7c17afa93a5a4d32848e68539054e336 2024-11-25T12:59:58,508 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/recovered.edits/482.seqid, newMaxSeqId=482, maxSeqId=1 2024-11-25T12:59:58,509 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. 2024-11-25T12:59:58,509 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1676): Region close journal for 6a5867ce2299ba674b3f042fd1fb41c6: Waiting for close lock at 1732539597173Running coprocessor pre-close hooks at 1732539597173Disabling compacts and flushes for region at 1732539597173Disabling writes for close at 1732539597173Obtaining lock to block concurrent updates at 1732539597173Preparing flush snapshotting stores in 6a5867ce2299ba674b3f042fd1fb41c6 at 1732539597173Finished memstore snapshotting TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=36720, getOffHeapSize=0, getCellsCount=300 at 1732539597173Flushing stores of TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6. at 1732539597174 (+1 ms)Flushing 6a5867ce2299ba674b3f042fd1fb41c6/A: creating writer at 1732539597174Flushing 6a5867ce2299ba674b3f042fd1fb41c6/A: appending metadata at 1732539597176 (+2 ms)Flushing 6a5867ce2299ba674b3f042fd1fb41c6/A: closing flushed file at 1732539597176Flushing 6a5867ce2299ba674b3f042fd1fb41c6/B: creating writer at 1732539597584 (+408 ms)Flushing 6a5867ce2299ba674b3f042fd1fb41c6/B: appending metadata at 1732539597586 (+2 ms)Flushing 6a5867ce2299ba674b3f042fd1fb41c6/B: closing flushed file at 1732539597586Flushing 6a5867ce2299ba674b3f042fd1fb41c6/C: creating writer at 1732539597992 (+406 ms)Flushing 6a5867ce2299ba674b3f042fd1fb41c6/C: appending metadata at 1732539597994 (+2 ms)Flushing 6a5867ce2299ba674b3f042fd1fb41c6/C: closing flushed file at 1732539597994Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@61d3f479: reopening flushed file at 1732539598403 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4da4e222: reopening flushed file at 1732539598407 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2f8b8c21: reopening flushed file at 1732539598410 (+3 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for 6a5867ce2299ba674b3f042fd1fb41c6 in 1240ms, sequenceid=479, compaction requested=true at 1732539598413 (+3 ms)Writing region close event to WAL at 1732539598506 (+93 ms)Running coprocessor post-close hooks at 1732539598509 (+3 ms)Closed at 1732539598509 2024-11-25T12:59:58,511 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] handler.UnassignRegionHandler(157): Closed 6a5867ce2299ba674b3f042fd1fb41c6 2024-11-25T12:59:58,511 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=129 updating hbase:meta row=6a5867ce2299ba674b3f042fd1fb41c6, regionState=CLOSED 2024-11-25T12:59:58,512 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=130, ppid=129, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 because future has completed 2024-11-25T12:59:58,514 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-25T12:59:58,514 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; CloseRegionProcedure 6a5867ce2299ba674b3f042fd1fb41c6, server=1ff8009e463e,35905,1732539449246 in 1.4930 sec 2024-11-25T12:59:58,516 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=129, resume processing ppid=128 2024-11-25T12:59:58,516 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, ppid=128, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6a5867ce2299ba674b3f042fd1fb41c6, UNASSIGN in 1.4960 sec 2024-11-25T12:59:58,517 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-25T12:59:58,517 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4990 sec 2024-11-25T12:59:58,518 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732539598518"}]},"ts":"1732539598518"} 2024-11-25T12:59:58,520 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-25T12:59:58,520 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-25T12:59:58,521 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5070 sec 2024-11-25T12:59:58,804 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-25T12:59:59,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-25T12:59:59,160 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:59:59,160 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-25T12:59:59,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T12:59:59,162 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=131, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T12:59:59,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-25T12:59:59,163 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=131, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T12:59:59,164 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6 2024-11-25T12:59:59,166 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A, FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B, FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C, FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/recovered.edits] 2024-11-25T12:59:59,168 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/3f1ca8c3fb2649d8aeb7adce85f22aee to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/3f1ca8c3fb2649d8aeb7adce85f22aee 2024-11-25T12:59:59,168 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/5973352d5fd9409d87b319e83a8fa2ef to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/5973352d5fd9409d87b319e83a8fa2ef 2024-11-25T12:59:59,169 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/c496d0305aa448f185149e528acc7960 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/A/c496d0305aa448f185149e528acc7960 2024-11-25T12:59:59,173 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/10c45f48837449e0b8bba90039e9dbe8 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/10c45f48837449e0b8bba90039e9dbe8 2024-11-25T12:59:59,174 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/41343c73340847dd8cdbfad20aa3b026 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/41343c73340847dd8cdbfad20aa3b026 2024-11-25T12:59:59,175 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/4e54d9acaa8848a894174a6fbd57c3f9 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/B/4e54d9acaa8848a894174a6fbd57c3f9 2024-11-25T12:59:59,176 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/e5bd2523eb184107a53db1c73772d4c8 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/e5bd2523eb184107a53db1c73772d4c8 2024-11-25T12:59:59,177 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/fc1d28cc5fc246cea052b594c3732c28 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/fc1d28cc5fc246cea052b594c3732c28 2024-11-25T12:59:59,178 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/fef5db707cc246158afde0c7c0020b47 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/C/fef5db707cc246158afde0c7c0020b47 2024-11-25T12:59:59,181 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/recovered.edits/482.seqid to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6/recovered.edits/482.seqid 2024-11-25T12:59:59,181 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/6a5867ce2299ba674b3f042fd1fb41c6 2024-11-25T12:59:59,181 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-25T12:59:59,183 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=131, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T12:59:59,185 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-25T12:59:59,187 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-25T12:59:59,188 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=131, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T12:59:59,188 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-25T12:59:59,188 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732539599188"}]},"ts":"9223372036854775807"} 2024-11-25T12:59:59,190 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-25T12:59:59,190 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 6a5867ce2299ba674b3f042fd1fb41c6, NAME => 'TestAcidGuarantees,,1732539573497.6a5867ce2299ba674b3f042fd1fb41c6.', STARTKEY => '', ENDKEY => ''}] 2024-11-25T12:59:59,190 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-25T12:59:59,190 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732539599190"}]},"ts":"9223372036854775807"} 2024-11-25T12:59:59,191 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-25T12:59:59,192 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=131, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T12:59:59,193 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 32 msec 2024-11-25T12:59:59,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-25T12:59:59,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-25T12:59:59,273 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:59:59,278 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=240 (was 242), OpenFileDescriptor=449 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1045 (was 1031) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=677 (was 899) 2024-11-25T12:59:59,289 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=240, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=1045, ProcessCount=11, AvailableMemoryMB=674 2024-11-25T12:59:59,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-25T12:59:59,291 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T12:59:59,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] procedure2.ProcedureExecutor(1139): Stored pid=132, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-25T12:59:59,293 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=132, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T12:59:59,293 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:59:59,293 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 132 2024-11-25T12:59:59,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-25T12:59:59,294 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=132, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T12:59:59,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742368_1544 (size=959) 2024-11-25T12:59:59,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-25T12:59:59,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-25T12:59:59,701 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a 2024-11-25T12:59:59,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742369_1545 (size=53) 2024-11-25T12:59:59,714 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:59:59,715 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 065eb16105c4aedcbfa7fb1ec2b14c86, disabling compactions & flushes 2024-11-25T12:59:59,715 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T12:59:59,715 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T12:59:59,715 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. after waiting 0 ms 2024-11-25T12:59:59,715 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T12:59:59,715 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T12:59:59,715 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 065eb16105c4aedcbfa7fb1ec2b14c86: Waiting for close lock at 1732539599715Disabling compacts and flushes for region at 1732539599715Disabling writes for close at 1732539599715Writing region close event to WAL at 1732539599715Closed at 1732539599715 2024-11-25T12:59:59,716 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=132, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T12:59:59,716 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732539599716"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732539599716"}]},"ts":"1732539599716"} 2024-11-25T12:59:59,719 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T12:59:59,720 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=132, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T12:59:59,720 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732539599720"}]},"ts":"1732539599720"} 2024-11-25T12:59:59,722 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-25T12:59:59,722 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=065eb16105c4aedcbfa7fb1ec2b14c86, ASSIGN}] 2024-11-25T12:59:59,723 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=065eb16105c4aedcbfa7fb1ec2b14c86, ASSIGN 2024-11-25T12:59:59,723 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=065eb16105c4aedcbfa7fb1ec2b14c86, ASSIGN; state=OFFLINE, location=1ff8009e463e,35905,1732539449246; forceNewPlan=false, retain=false 2024-11-25T12:59:59,874 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=133 updating hbase:meta row=065eb16105c4aedcbfa7fb1ec2b14c86, regionState=OPENING, regionLocation=1ff8009e463e,35905,1732539449246 2024-11-25T12:59:59,875 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=065eb16105c4aedcbfa7fb1ec2b14c86, ASSIGN because future has completed 2024-11-25T12:59:59,876 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; OpenRegionProcedure 065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246}] 2024-11-25T12:59:59,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-25T13:00:00,030 INFO [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:00,030 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(7752): Opening region: {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} 2024-11-25T13:00:00,030 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:00,030 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T13:00:00,030 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(7794): checking encryption for 065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:00,030 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(7797): checking classloading for 065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:00,031 INFO [StoreOpener-065eb16105c4aedcbfa7fb1ec2b14c86-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:00,032 INFO [StoreOpener-065eb16105c4aedcbfa7fb1ec2b14c86-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 065eb16105c4aedcbfa7fb1ec2b14c86 columnFamilyName A 2024-11-25T13:00:00,032 DEBUG [StoreOpener-065eb16105c4aedcbfa7fb1ec2b14c86-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:00,033 INFO [StoreOpener-065eb16105c4aedcbfa7fb1ec2b14c86-1 {}] regionserver.HStore(327): Store=065eb16105c4aedcbfa7fb1ec2b14c86/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T13:00:00,033 INFO [StoreOpener-065eb16105c4aedcbfa7fb1ec2b14c86-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:00,033 INFO [StoreOpener-065eb16105c4aedcbfa7fb1ec2b14c86-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 065eb16105c4aedcbfa7fb1ec2b14c86 columnFamilyName B 2024-11-25T13:00:00,033 DEBUG [StoreOpener-065eb16105c4aedcbfa7fb1ec2b14c86-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:00,034 INFO [StoreOpener-065eb16105c4aedcbfa7fb1ec2b14c86-1 {}] regionserver.HStore(327): Store=065eb16105c4aedcbfa7fb1ec2b14c86/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T13:00:00,034 INFO [StoreOpener-065eb16105c4aedcbfa7fb1ec2b14c86-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:00,035 INFO [StoreOpener-065eb16105c4aedcbfa7fb1ec2b14c86-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 065eb16105c4aedcbfa7fb1ec2b14c86 columnFamilyName C 2024-11-25T13:00:00,035 DEBUG [StoreOpener-065eb16105c4aedcbfa7fb1ec2b14c86-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:00,035 INFO [StoreOpener-065eb16105c4aedcbfa7fb1ec2b14c86-1 {}] regionserver.HStore(327): Store=065eb16105c4aedcbfa7fb1ec2b14c86/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T13:00:00,035 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1038): replaying wal for 065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:00,036 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:00,036 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:00,036 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1048): stopping wal replay for 065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:00,036 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1060): Cleaning up temporary data for 065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:00,038 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-25T13:00:00,039 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1093): writing seq id for 065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:00,041 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T13:00:00,041 INFO [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1114): Opened 065eb16105c4aedcbfa7fb1ec2b14c86; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66792086, jitterRate=-0.004720360040664673}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-25T13:00:00,041 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:00,042 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1006): Region open journal for 065eb16105c4aedcbfa7fb1ec2b14c86: Running coprocessor pre-open hook at 1732539600030Writing region info on filesystem at 1732539600030Initializing all the Stores at 1732539600031 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732539600031Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732539600031Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732539600031Cleaning up temporary data from old regions at 1732539600036 (+5 ms)Running coprocessor post-open hooks at 1732539600041 (+5 ms)Region opened successfully at 1732539600042 (+1 ms) 2024-11-25T13:00:00,043 INFO [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., pid=134, masterSystemTime=1732539600027 2024-11-25T13:00:00,044 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:00,044 INFO [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:00,045 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=133 updating hbase:meta row=065eb16105c4aedcbfa7fb1ec2b14c86, regionState=OPEN, openSeqNum=2, regionLocation=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:00,046 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=134, ppid=133, state=RUNNABLE, hasLock=false; OpenRegionProcedure 065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 because future has completed 2024-11-25T13:00:00,048 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-25T13:00:00,048 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; OpenRegionProcedure 065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 in 170 msec 2024-11-25T13:00:00,049 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=133, resume processing ppid=132 2024-11-25T13:00:00,049 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, ppid=132, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=065eb16105c4aedcbfa7fb1ec2b14c86, ASSIGN in 326 msec 2024-11-25T13:00:00,049 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=132, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T13:00:00,050 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732539600049"}]},"ts":"1732539600049"} 2024-11-25T13:00:00,051 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-25T13:00:00,051 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=132, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T13:00:00,052 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 760 msec 2024-11-25T13:00:00,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-25T13:00:00,430 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-25T13:00:00,431 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53954261, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T13:00:00,431 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ff8009e463e,46289,-1 for getting cluster id 2024-11-25T13:00:00,432 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T13:00:00,435 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb972fd5-cdcb-422b-acfe-41fc80d301d9' 2024-11-25T13:00:00,435 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T13:00:00,435 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb972fd5-cdcb-422b-acfe-41fc80d301d9" 2024-11-25T13:00:00,435 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6062603, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T13:00:00,435 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ff8009e463e,46289,-1] 2024-11-25T13:00:00,436 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T13:00:00,436 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T13:00:00,437 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50760, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T13:00:00,437 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3dad87c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T13:00:00,437 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T13:00:00,438 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,35905,1732539449246, seqNum=-1] 2024-11-25T13:00:00,438 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T13:00:00,439 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33720, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T13:00:00,440 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-25T13:00:00,440 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 1ff8009e463e,46289,1732539448369 2024-11-25T13:00:00,440 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3cd04a01 2024-11-25T13:00:00,440 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-25T13:00:00,441 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50768, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-25T13:00:00,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-25T13:00:00,442 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T13:00:00,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-25T13:00:00,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742370_1546 (size=995) 2024-11-25T13:00:00,864 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-25T13:00:00,864 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-25T13:00:00,867 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-25T13:00:00,870 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=065eb16105c4aedcbfa7fb1ec2b14c86, REOPEN/MOVE}] 2024-11-25T13:00:00,872 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=065eb16105c4aedcbfa7fb1ec2b14c86, REOPEN/MOVE 2024-11-25T13:00:00,874 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=065eb16105c4aedcbfa7fb1ec2b14c86, regionState=CLOSING, regionLocation=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:00,875 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=065eb16105c4aedcbfa7fb1ec2b14c86, REOPEN/MOVE because future has completed 2024-11-25T13:00:00,875 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-25T13:00:00,875 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; CloseRegionProcedure 065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246}] 2024-11-25T13:00:01,028 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(122): Close 065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:01,028 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-25T13:00:01,028 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1722): Closing 065eb16105c4aedcbfa7fb1ec2b14c86, disabling compactions & flushes 2024-11-25T13:00:01,028 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:01,028 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:01,028 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. after waiting 0 ms 2024-11-25T13:00:01,028 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:01,031 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-25T13:00:01,031 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:01,032 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1676): Region close journal for 065eb16105c4aedcbfa7fb1ec2b14c86: Waiting for close lock at 1732539601028Running coprocessor pre-close hooks at 1732539601028Disabling compacts and flushes for region at 1732539601028Disabling writes for close at 1732539601028Writing region close event to WAL at 1732539601029 (+1 ms)Running coprocessor post-close hooks at 1732539601031 (+2 ms)Closed at 1732539601031 2024-11-25T13:00:01,032 WARN [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegionServer(3299): Not adding moved region record: 065eb16105c4aedcbfa7fb1ec2b14c86 to self. 2024-11-25T13:00:01,033 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(157): Closed 065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:01,034 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=065eb16105c4aedcbfa7fb1ec2b14c86, regionState=CLOSED 2024-11-25T13:00:01,035 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=138, ppid=137, state=RUNNABLE, hasLock=false; CloseRegionProcedure 065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 because future has completed 2024-11-25T13:00:01,037 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-25T13:00:01,037 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; CloseRegionProcedure 065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 in 161 msec 2024-11-25T13:00:01,037 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=065eb16105c4aedcbfa7fb1ec2b14c86, REOPEN/MOVE; state=CLOSED, location=1ff8009e463e,35905,1732539449246; forceNewPlan=false, retain=true 2024-11-25T13:00:01,188 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=065eb16105c4aedcbfa7fb1ec2b14c86, regionState=OPENING, regionLocation=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:01,190 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=065eb16105c4aedcbfa7fb1ec2b14c86, REOPEN/MOVE because future has completed 2024-11-25T13:00:01,190 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=137, state=RUNNABLE, hasLock=false; OpenRegionProcedure 065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246}] 2024-11-25T13:00:01,346 INFO [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:01,346 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(7752): Opening region: {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} 2024-11-25T13:00:01,347 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:01,347 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T13:00:01,347 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(7794): checking encryption for 065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:01,347 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(7797): checking classloading for 065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:01,351 INFO [StoreOpener-065eb16105c4aedcbfa7fb1ec2b14c86-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:01,352 INFO [StoreOpener-065eb16105c4aedcbfa7fb1ec2b14c86-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 065eb16105c4aedcbfa7fb1ec2b14c86 columnFamilyName A 2024-11-25T13:00:01,353 DEBUG [StoreOpener-065eb16105c4aedcbfa7fb1ec2b14c86-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:01,355 INFO [StoreOpener-065eb16105c4aedcbfa7fb1ec2b14c86-1 {}] regionserver.HStore(327): Store=065eb16105c4aedcbfa7fb1ec2b14c86/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T13:00:01,355 INFO [StoreOpener-065eb16105c4aedcbfa7fb1ec2b14c86-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:01,356 INFO [StoreOpener-065eb16105c4aedcbfa7fb1ec2b14c86-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 065eb16105c4aedcbfa7fb1ec2b14c86 columnFamilyName B 2024-11-25T13:00:01,356 DEBUG [StoreOpener-065eb16105c4aedcbfa7fb1ec2b14c86-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:01,356 INFO [StoreOpener-065eb16105c4aedcbfa7fb1ec2b14c86-1 {}] regionserver.HStore(327): Store=065eb16105c4aedcbfa7fb1ec2b14c86/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T13:00:01,356 INFO [StoreOpener-065eb16105c4aedcbfa7fb1ec2b14c86-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:01,357 INFO [StoreOpener-065eb16105c4aedcbfa7fb1ec2b14c86-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 065eb16105c4aedcbfa7fb1ec2b14c86 columnFamilyName C 2024-11-25T13:00:01,357 DEBUG [StoreOpener-065eb16105c4aedcbfa7fb1ec2b14c86-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:01,357 INFO [StoreOpener-065eb16105c4aedcbfa7fb1ec2b14c86-1 {}] regionserver.HStore(327): Store=065eb16105c4aedcbfa7fb1ec2b14c86/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T13:00:01,357 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1038): replaying wal for 065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:01,358 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:01,360 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:01,361 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1048): stopping wal replay for 065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:01,361 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1060): Cleaning up temporary data for 065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:01,361 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-25T13:00:01,363 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1093): writing seq id for 065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:01,364 INFO [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1114): Opened 065eb16105c4aedcbfa7fb1ec2b14c86; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58871764, jitterRate=-0.12274235486984253}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-25T13:00:01,364 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:01,365 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1006): Region open journal for 065eb16105c4aedcbfa7fb1ec2b14c86: Running coprocessor pre-open hook at 1732539601347Writing region info on filesystem at 1732539601347Initializing all the Stores at 1732539601350 (+3 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732539601350Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732539601350Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732539601351 (+1 ms)Cleaning up temporary data from old regions at 1732539601361 (+10 ms)Running coprocessor post-open hooks at 1732539601364 (+3 ms)Region opened successfully at 1732539601365 (+1 ms) 2024-11-25T13:00:01,365 INFO [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., pid=139, masterSystemTime=1732539601343 2024-11-25T13:00:01,367 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:01,367 INFO [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:01,368 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=065eb16105c4aedcbfa7fb1ec2b14c86, regionState=OPEN, openSeqNum=5, regionLocation=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:01,369 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=137, state=RUNNABLE, hasLock=false; OpenRegionProcedure 065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 because future has completed 2024-11-25T13:00:01,371 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=137 2024-11-25T13:00:01,371 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=137, state=SUCCESS, hasLock=false; OpenRegionProcedure 065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 in 180 msec 2024-11-25T13:00:01,373 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=137, resume processing ppid=136 2024-11-25T13:00:01,374 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, ppid=136, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=065eb16105c4aedcbfa7fb1ec2b14c86, REOPEN/MOVE in 501 msec 2024-11-25T13:00:01,391 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-25T13:00:01,391 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 507 msec 2024-11-25T13:00:01,401 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 952 msec 2024-11-25T13:00:01,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-25T13:00:01,402 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-25T13:00:01,403 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42a3e32a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T13:00:01,403 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ff8009e463e,46289,-1 for getting cluster id 2024-11-25T13:00:01,403 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T13:00:01,405 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb972fd5-cdcb-422b-acfe-41fc80d301d9' 2024-11-25T13:00:01,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T13:00:01,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb972fd5-cdcb-422b-acfe-41fc80d301d9" 2024-11-25T13:00:01,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6de53b9a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T13:00:01,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ff8009e463e,46289,-1] 2024-11-25T13:00:01,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T13:00:01,406 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T13:00:01,407 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50786, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T13:00:01,408 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65c4ab6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T13:00:01,408 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@556fc022, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T13:00:01,408 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ff8009e463e,46289,-1 for getting cluster id 2024-11-25T13:00:01,409 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T13:00:01,409 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb972fd5-cdcb-422b-acfe-41fc80d301d9' 2024-11-25T13:00:01,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T13:00:01,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb972fd5-cdcb-422b-acfe-41fc80d301d9" 2024-11-25T13:00:01,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37936e8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T13:00:01,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ff8009e463e,46289,-1] 2024-11-25T13:00:01,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T13:00:01,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T13:00:01,411 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50804, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T13:00:01,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@123d950a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T13:00:01,412 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77590f25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T13:00:01,412 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ff8009e463e,46289,-1 for getting cluster id 2024-11-25T13:00:01,413 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T13:00:01,413 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb972fd5-cdcb-422b-acfe-41fc80d301d9' 2024-11-25T13:00:01,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T13:00:01,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb972fd5-cdcb-422b-acfe-41fc80d301d9" 2024-11-25T13:00:01,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6942f0c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T13:00:01,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ff8009e463e,46289,-1] 2024-11-25T13:00:01,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T13:00:01,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T13:00:01,415 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50830, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T13:00:01,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a4aa25c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T13:00:01,423 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@99b73f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T13:00:01,424 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ff8009e463e,46289,-1 for getting cluster id 2024-11-25T13:00:01,424 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T13:00:01,425 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb972fd5-cdcb-422b-acfe-41fc80d301d9' 2024-11-25T13:00:01,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T13:00:01,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb972fd5-cdcb-422b-acfe-41fc80d301d9" 2024-11-25T13:00:01,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b8501c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T13:00:01,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ff8009e463e,46289,-1] 2024-11-25T13:00:01,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T13:00:01,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T13:00:01,426 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50850, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T13:00:01,430 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f5207d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T13:00:01,435 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@af69a6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T13:00:01,435 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ff8009e463e,46289,-1 for getting cluster id 2024-11-25T13:00:01,436 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T13:00:01,436 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb972fd5-cdcb-422b-acfe-41fc80d301d9' 2024-11-25T13:00:01,437 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T13:00:01,437 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb972fd5-cdcb-422b-acfe-41fc80d301d9" 2024-11-25T13:00:01,437 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49622aa1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T13:00:01,437 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ff8009e463e,46289,-1] 2024-11-25T13:00:01,437 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T13:00:01,437 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T13:00:01,438 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50862, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T13:00:01,438 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e658cf2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T13:00:01,439 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cb7cff8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T13:00:01,439 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ff8009e463e,46289,-1 for getting cluster id 2024-11-25T13:00:01,439 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T13:00:01,440 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb972fd5-cdcb-422b-acfe-41fc80d301d9' 2024-11-25T13:00:01,440 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T13:00:01,440 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb972fd5-cdcb-422b-acfe-41fc80d301d9" 2024-11-25T13:00:01,440 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18be61ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T13:00:01,441 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ff8009e463e,46289,-1] 2024-11-25T13:00:01,441 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T13:00:01,441 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T13:00:01,441 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50868, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T13:00:01,442 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@363fd33e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T13:00:01,443 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56ae597b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T13:00:01,443 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ff8009e463e,46289,-1 for getting cluster id 2024-11-25T13:00:01,443 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T13:00:01,444 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb972fd5-cdcb-422b-acfe-41fc80d301d9' 2024-11-25T13:00:01,444 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T13:00:01,444 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb972fd5-cdcb-422b-acfe-41fc80d301d9" 2024-11-25T13:00:01,444 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8acfd14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T13:00:01,444 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ff8009e463e,46289,-1] 2024-11-25T13:00:01,444 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T13:00:01,445 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T13:00:01,445 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50892, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T13:00:01,446 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58c0e671, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T13:00:01,447 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39174ad3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T13:00:01,447 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ff8009e463e,46289,-1 for getting cluster id 2024-11-25T13:00:01,448 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T13:00:01,448 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb972fd5-cdcb-422b-acfe-41fc80d301d9' 2024-11-25T13:00:01,448 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T13:00:01,449 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb972fd5-cdcb-422b-acfe-41fc80d301d9" 2024-11-25T13:00:01,449 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c3c305b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T13:00:01,449 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ff8009e463e,46289,-1] 2024-11-25T13:00:01,449 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T13:00:01,449 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T13:00:01,450 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50914, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T13:00:01,450 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5878fc70, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T13:00:01,451 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e8fc71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T13:00:01,451 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ff8009e463e,46289,-1 for getting cluster id 2024-11-25T13:00:01,451 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T13:00:01,452 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb972fd5-cdcb-422b-acfe-41fc80d301d9' 2024-11-25T13:00:01,452 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T13:00:01,452 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb972fd5-cdcb-422b-acfe-41fc80d301d9" 2024-11-25T13:00:01,452 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@691d898e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T13:00:01,452 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ff8009e463e,46289,-1] 2024-11-25T13:00:01,452 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T13:00:01,453 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T13:00:01,453 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50924, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T13:00:01,454 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a21001d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T13:00:01,455 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dc96918, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T13:00:01,455 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ff8009e463e,46289,-1 for getting cluster id 2024-11-25T13:00:01,455 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T13:00:01,457 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb972fd5-cdcb-422b-acfe-41fc80d301d9' 2024-11-25T13:00:01,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T13:00:01,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb972fd5-cdcb-422b-acfe-41fc80d301d9" 2024-11-25T13:00:01,458 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@340ab49a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T13:00:01,458 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ff8009e463e,46289,-1] 2024-11-25T13:00:01,458 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T13:00:01,458 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T13:00:01,459 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50934, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T13:00:01,459 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e125faa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T13:00:01,460 DEBUG [Thread-2438 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T13:00:01,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,35905,1732539449246, seqNum=-1] 2024-11-25T13:00:01,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T13:00:01,461 DEBUG [Thread-2436 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T13:00:01,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,35905,1732539449246, seqNum=-1] 2024-11-25T13:00:01,461 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33734, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T13:00:01,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T13:00:01,462 DEBUG [Thread-2453 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T13:00:01,462 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33740, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T13:00:01,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5] 2024-11-25T13:00:01,463 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,35905,1732539449246, seqNum=-1] 2024-11-25T13:00:01,463 DEBUG [Thread-2442 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T13:00:01,463 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T13:00:01,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5] 2024-11-25T13:00:01,464 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33742, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T13:00:01,465 DEBUG [Thread-2449 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T13:00:01,466 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,35905,1732539449246, seqNum=-1] 2024-11-25T13:00:01,466 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T13:00:01,467 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5] 2024-11-25T13:00:01,468 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33748, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T13:00:01,468 DEBUG [Thread-2440 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T13:00:01,469 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,35905,1732539449246, seqNum=-1] 2024-11-25T13:00:01,469 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,35905,1732539449246, seqNum=-1] 2024-11-25T13:00:01,469 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T13:00:01,469 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T13:00:01,471 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33778, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T13:00:01,471 DEBUG [Thread-2434 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T13:00:01,472 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5] 2024-11-25T13:00:01,473 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5] 2024-11-25T13:00:01,473 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,35905,1732539449246, seqNum=-1] 2024-11-25T13:00:01,473 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33764, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T13:00:01,473 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T13:00:01,474 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5] 2024-11-25T13:00:01,475 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33782, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T13:00:01,476 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5] 2024-11-25T13:00:01,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:01,478 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T13:00:01,478 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 065eb16105c4aedcbfa7fb1ec2b14c86 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-25T13:00:01,479 DEBUG [Thread-2445 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T13:00:01,480 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,35905,1732539449246, seqNum=-1] 2024-11-25T13:00:01,481 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T13:00:01,483 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33796, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T13:00:01,492 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5] 2024-11-25T13:00:01,494 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T13:00:01,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] procedure2.ProcedureExecutor(1139): Stored pid=140, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees 2024-11-25T13:00:01,495 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=140, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T13:00:01,496 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=140, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T13:00:01,496 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T13:00:01,499 DEBUG [Thread-2451 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T13:00:01,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-25T13:00:01,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,35905,1732539449246, seqNum=-1] 2024-11-25T13:00:01,501 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T13:00:01,501 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33800, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T13:00:01,507 DEBUG [Thread-2447 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T13:00:01,508 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,35905,1732539449246, seqNum=-1] 2024-11-25T13:00:01,508 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T13:00:01,512 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5] 2024-11-25T13:00:01,513 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33814, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T13:00:01,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:01,516 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5] 2024-11-25T13:00:01,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33764 deadline: 1732539661508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:01,517 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:01,517 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:01,517 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:01,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:01,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33782 deadline: 1732539661513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:01,520 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:01,520 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:01,520 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:01,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:01,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1732539661517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:01,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:01,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:01,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33740 deadline: 1732539661517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:01,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:01,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:01,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:01,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:01,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:01,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:01,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33778 deadline: 1732539661526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:01,537 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:01,537 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:01,537 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:01,557 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125a87d6582de754e589b481d1f2e2acc81_065eb16105c4aedcbfa7fb1ec2b14c86 is 50, key is test_row_1/A:col10/1732539601477/Put/seqid=0 2024-11-25T13:00:01,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742371_1547 (size=9714) 2024-11-25T13:00:01,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-25T13:00:01,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:01,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33764 deadline: 1732539661620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:01,621 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:01,621 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:01,621 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:01,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:01,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33782 deadline: 1732539661630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:01,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:01,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33740 deadline: 1732539661630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:01,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:01,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:01,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1732539661630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:01,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:01,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:01,635 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:01,636 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:01,636 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:01,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:01,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:01,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:01,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:01,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33778 deadline: 1732539661640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:01,644 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:01,644 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:01,644 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:01,651 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-25T13:00:01,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:01,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. as already flushing 2024-11-25T13:00:01,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:01,651 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:01,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:01,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:01,812 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-25T13:00:01,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:01,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. as already flushing 2024-11-25T13:00:01,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:01,813 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:01,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:01,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:01,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-25T13:00:01,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:01,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33764 deadline: 1732539661830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:01,833 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:01,833 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:01,833 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:01,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:01,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33740 deadline: 1732539661840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:01,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:01,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:01,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:01,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:01,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33782 deadline: 1732539661840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:01,842 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:01,842 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:01,842 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:01,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:01,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1732539661840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:01,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:01,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:01,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:01,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:01,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33778 deadline: 1732539661849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:01,851 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:01,851 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:01,851 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:01,967 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-25T13:00:01,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:01,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. as already flushing 2024-11-25T13:00:01,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:01,967 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:01,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:01,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:02,000 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:02,009 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125a87d6582de754e589b481d1f2e2acc81_065eb16105c4aedcbfa7fb1ec2b14c86 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125a87d6582de754e589b481d1f2e2acc81_065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:02,011 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/c8fa4d5e1cff4ce7af133036fcaa5c96, store: [table=TestAcidGuarantees family=A region=065eb16105c4aedcbfa7fb1ec2b14c86] 2024-11-25T13:00:02,011 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/c8fa4d5e1cff4ce7af133036fcaa5c96 is 175, key is test_row_1/A:col10/1732539601477/Put/seqid=0 2024-11-25T13:00:02,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742372_1548 (size=22361) 2024-11-25T13:00:02,033 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=18, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/c8fa4d5e1cff4ce7af133036fcaa5c96 2024-11-25T13:00:02,056 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/a85fb91fd49b47f3be87d55cc4ce7d09 is 50, key is test_row_1/B:col10/1732539601477/Put/seqid=0 2024-11-25T13:00:02,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742373_1549 (size=9657) 2024-11-25T13:00:02,086 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/a85fb91fd49b47f3be87d55cc4ce7d09 2024-11-25T13:00:02,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-25T13:00:02,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:02,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. as already flushing 2024-11-25T13:00:02,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:02,121 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:02,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:02,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:02,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-25T13:00:02,141 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/9ed428103bca4174b11c33b713045b89 is 50, key is test_row_1/C:col10/1732539601477/Put/seqid=0 2024-11-25T13:00:02,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:02,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33764 deadline: 1732539662140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:02,143 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:02,143 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:02,143 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:02,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742374_1550 (size=9657) 2024-11-25T13:00:02,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:02,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33740 deadline: 1732539662150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:02,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:02,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33782 deadline: 1732539662150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:02,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:02,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:02,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:02,152 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:02,152 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:02,152 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:02,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:02,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1732539662150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:02,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:02,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:02,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:02,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:02,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33778 deadline: 1732539662160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:02,162 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:02,162 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:02,162 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:02,275 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-25T13:00:02,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:02,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. as already flushing 2024-11-25T13:00:02,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:02,276 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:02,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:02,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:02,436 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-25T13:00:02,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:02,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. as already flushing 2024-11-25T13:00:02,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:02,436 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:02,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:02,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:02,550 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/9ed428103bca4174b11c33b713045b89 2024-11-25T13:00:02,557 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/c8fa4d5e1cff4ce7af133036fcaa5c96 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/c8fa4d5e1cff4ce7af133036fcaa5c96 2024-11-25T13:00:02,564 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/c8fa4d5e1cff4ce7af133036fcaa5c96, entries=100, sequenceid=18, filesize=21.8 K 2024-11-25T13:00:02,565 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/a85fb91fd49b47f3be87d55cc4ce7d09 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/a85fb91fd49b47f3be87d55cc4ce7d09 2024-11-25T13:00:02,569 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/a85fb91fd49b47f3be87d55cc4ce7d09, entries=100, sequenceid=18, filesize=9.4 K 2024-11-25T13:00:02,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/9ed428103bca4174b11c33b713045b89 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/9ed428103bca4174b11c33b713045b89 2024-11-25T13:00:02,583 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/9ed428103bca4174b11c33b713045b89, entries=100, sequenceid=18, filesize=9.4 K 2024-11-25T13:00:02,588 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 065eb16105c4aedcbfa7fb1ec2b14c86 in 1110ms, sequenceid=18, compaction requested=false 2024-11-25T13:00:02,588 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 065eb16105c4aedcbfa7fb1ec2b14c86: 2024-11-25T13:00:02,594 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-25T13:00:02,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:02,594 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2902): Flushing 065eb16105c4aedcbfa7fb1ec2b14c86 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-25T13:00:02,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411252ac9b8682fe04ca49f58eb13ce359fd4_065eb16105c4aedcbfa7fb1ec2b14c86 is 50, key is test_row_0/A:col10/1732539601514/Put/seqid=0 2024-11-25T13:00:02,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-25T13:00:02,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:02,652 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. as already flushing 2024-11-25T13:00:02,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:02,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1732539662676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:02,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:02,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33778 deadline: 1732539662676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:02,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742375_1551 (size=12154) 2024-11-25T13:00:02,681 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:02,681 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:02,681 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:02,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:02,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:02,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:02,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:02,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33764 deadline: 1732539662678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:02,686 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:02,686 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:02,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:02,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:02,686 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:02,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33740 deadline: 1732539662679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:02,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33782 deadline: 1732539662679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:02,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:02,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:02,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:02,687 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:02,687 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:02,687 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:02,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:02,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33764 deadline: 1732539662790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:02,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:02,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:02,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:02,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33740 deadline: 1732539662790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:02,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:02,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:02,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:02,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:02,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:02,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1732539662791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:02,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:02,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:02,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:02,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:02,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33782 deadline: 1732539662791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:02,796 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:02,796 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:02,796 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:03,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:03,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33740 deadline: 1732539663002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:03,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:03,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33782 deadline: 1732539663002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:03,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:03,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:03,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:03,007 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:03,007 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:03,007 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:03,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:03,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33764 deadline: 1732539663006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:03,011 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:03,011 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:03,011 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:03,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:03,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1732539663006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:03,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:03,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:03,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:03,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:03,106 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411252ac9b8682fe04ca49f58eb13ce359fd4_065eb16105c4aedcbfa7fb1ec2b14c86 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411252ac9b8682fe04ca49f58eb13ce359fd4_065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:03,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/628f46655ebf45778131f9f8b76caf88, store: [table=TestAcidGuarantees family=A region=065eb16105c4aedcbfa7fb1ec2b14c86] 2024-11-25T13:00:03,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/628f46655ebf45778131f9f8b76caf88 is 175, key is test_row_0/A:col10/1732539601514/Put/seqid=0 2024-11-25T13:00:03,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742376_1552 (size=30955) 2024-11-25T13:00:03,149 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/628f46655ebf45778131f9f8b76caf88 2024-11-25T13:00:03,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/b4882718154f4426abb1ed8dbe384cd9 is 50, key is test_row_0/B:col10/1732539601514/Put/seqid=0 2024-11-25T13:00:03,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742377_1553 (size=12001) 2024-11-25T13:00:03,177 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/b4882718154f4426abb1ed8dbe384cd9 2024-11-25T13:00:03,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/f8ff43fb7780480e8508e5edf99d76d2 is 50, key is test_row_0/C:col10/1732539601514/Put/seqid=0 2024-11-25T13:00:03,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742378_1554 (size=12001) 2024-11-25T13:00:03,191 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/f8ff43fb7780480e8508e5edf99d76d2 2024-11-25T13:00:03,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/628f46655ebf45778131f9f8b76caf88 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/628f46655ebf45778131f9f8b76caf88 2024-11-25T13:00:03,207 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/628f46655ebf45778131f9f8b76caf88, entries=150, sequenceid=40, filesize=30.2 K 2024-11-25T13:00:03,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/b4882718154f4426abb1ed8dbe384cd9 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/b4882718154f4426abb1ed8dbe384cd9 2024-11-25T13:00:03,218 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/b4882718154f4426abb1ed8dbe384cd9, entries=150, sequenceid=40, filesize=11.7 K 2024-11-25T13:00:03,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/f8ff43fb7780480e8508e5edf99d76d2 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/f8ff43fb7780480e8508e5edf99d76d2 2024-11-25T13:00:03,227 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/f8ff43fb7780480e8508e5edf99d76d2, entries=150, sequenceid=40, filesize=11.7 K 2024-11-25T13:00:03,228 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 065eb16105c4aedcbfa7fb1ec2b14c86 in 634ms, sequenceid=40, compaction requested=false 2024-11-25T13:00:03,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2603): Flush status journal for 065eb16105c4aedcbfa7fb1ec2b14c86: 2024-11-25T13:00:03,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:03,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=141 2024-11-25T13:00:03,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4169): Remote procedure done, pid=141 2024-11-25T13:00:03,231 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-11-25T13:00:03,231 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7340 sec 2024-11-25T13:00:03,233 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees in 1.7380 sec 2024-11-25T13:00:03,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:03,312 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T13:00:03,312 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 065eb16105c4aedcbfa7fb1ec2b14c86 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-25T13:00:03,329 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125e8996a5b2a3343bfbcb7dd493fb72075_065eb16105c4aedcbfa7fb1ec2b14c86 is 50, key is test_row_0/A:col10/1732539602677/Put/seqid=0 2024-11-25T13:00:03,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742379_1555 (size=12154) 2024-11-25T13:00:03,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:03,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:03,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33764 deadline: 1732539663371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:03,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1732539663373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:03,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:03,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:03,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:03,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33782 deadline: 1732539663375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:03,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:03,379 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:03,379 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:03,379 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:03,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:03,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:03,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:03,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:03,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33740 deadline: 1732539663376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:03,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:03,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:03,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:03,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:03,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1732539663491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:03,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:03,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33782 deadline: 1732539663491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:03,493 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:03,493 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:03,493 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:03,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:03,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:03,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:03,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:03,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33764 deadline: 1732539663492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:03,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:03,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33740 deadline: 1732539663492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:03,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:03,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:03,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:03,495 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:03,495 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:03,495 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:03,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-25T13:00:03,654 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T13:00:03,655 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T13:00:03,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] procedure2.ProcedureExecutor(1139): Stored pid=142, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees 2024-11-25T13:00:03,657 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=142, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T13:00:03,658 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=142, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T13:00:03,658 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T13:00:03,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-25T13:00:03,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:03,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33778 deadline: 1732539663690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:03,693 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 1ff8009e463e,35905,1732539449246 for 'test_row_2' in 065eb16105c4aedcbfa7fb1ec2b14c86 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T13:00:03,694 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:03,694 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:03,694 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:03,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:03,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33740 deadline: 1732539663700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:03,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:03,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:03,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:03,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:03,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33764 deadline: 1732539663700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:03,705 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:03,705 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:03,705 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:03,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:03,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:03,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33782 deadline: 1732539663705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:03,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1732539663705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:03,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:03,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:03,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:03,710 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:03,710 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:03,710 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:03,746 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:03,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-25T13:00:03,784 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125e8996a5b2a3343bfbcb7dd493fb72075_065eb16105c4aedcbfa7fb1ec2b14c86 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125e8996a5b2a3343bfbcb7dd493fb72075_065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:03,796 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/71f8c18f61bd4fab838ff17930b4a166, store: [table=TestAcidGuarantees family=A region=065eb16105c4aedcbfa7fb1ec2b14c86] 2024-11-25T13:00:03,800 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/71f8c18f61bd4fab838ff17930b4a166 is 175, key is test_row_0/A:col10/1732539602677/Put/seqid=0 2024-11-25T13:00:03,819 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-25T13:00:03,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:03,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. as already flushing 2024-11-25T13:00:03,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:03,820 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:03,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:03,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:03,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742380_1556 (size=30955) 2024-11-25T13:00:03,828 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=55, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/71f8c18f61bd4fab838ff17930b4a166 2024-11-25T13:00:03,862 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/b186d3dc5e1048e4a094684416935727 is 50, key is test_row_0/B:col10/1732539602677/Put/seqid=0 2024-11-25T13:00:03,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742381_1557 (size=12001) 2024-11-25T13:00:03,973 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-25T13:00:03,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:03,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. as already flushing 2024-11-25T13:00:03,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:03,973 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:03,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:03,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:03,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-25T13:00:04,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:04,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33740 deadline: 1732539664010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:04,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:04,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:04,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:04,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:04,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33764 deadline: 1732539664010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:04,015 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:04,015 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:04,015 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:04,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:04,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33782 deadline: 1732539664024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:04,028 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:04,029 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:04,029 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:04,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:04,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1732539664024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:04,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:04,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:04,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:04,036 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T13:00:04,128 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-25T13:00:04,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:04,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. as already flushing 2024-11-25T13:00:04,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:04,128 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:04,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:04,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:04,285 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-25T13:00:04,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:04,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. as already flushing 2024-11-25T13:00:04,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:04,285 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:04,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:04,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:04,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-25T13:00:04,294 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/b186d3dc5e1048e4a094684416935727 2024-11-25T13:00:04,365 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/897730c0e49945dda8098bafb3d628d8 is 50, key is test_row_0/C:col10/1732539602677/Put/seqid=0 2024-11-25T13:00:04,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742382_1558 (size=12001) 2024-11-25T13:00:04,442 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/897730c0e49945dda8098bafb3d628d8 2024-11-25T13:00:04,442 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-25T13:00:04,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:04,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. as already flushing 2024-11-25T13:00:04,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:04,443 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:04,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:04,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:04,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/71f8c18f61bd4fab838ff17930b4a166 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/71f8c18f61bd4fab838ff17930b4a166 2024-11-25T13:00:04,486 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/71f8c18f61bd4fab838ff17930b4a166, entries=150, sequenceid=55, filesize=30.2 K 2024-11-25T13:00:04,486 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/b186d3dc5e1048e4a094684416935727 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/b186d3dc5e1048e4a094684416935727 2024-11-25T13:00:04,491 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/b186d3dc5e1048e4a094684416935727, entries=150, sequenceid=55, filesize=11.7 K 2024-11-25T13:00:04,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/897730c0e49945dda8098bafb3d628d8 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/897730c0e49945dda8098bafb3d628d8 2024-11-25T13:00:04,498 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/897730c0e49945dda8098bafb3d628d8, entries=150, sequenceid=55, filesize=11.7 K 2024-11-25T13:00:04,500 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 065eb16105c4aedcbfa7fb1ec2b14c86 in 1188ms, sequenceid=55, compaction requested=true 2024-11-25T13:00:04,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 065eb16105c4aedcbfa7fb1ec2b14c86: 2024-11-25T13:00:04,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 065eb16105c4aedcbfa7fb1ec2b14c86:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T13:00:04,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T13:00:04,500 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T13:00:04,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 065eb16105c4aedcbfa7fb1ec2b14c86:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T13:00:04,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T13:00:04,500 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T13:00:04,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 065eb16105c4aedcbfa7fb1ec2b14c86:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T13:00:04,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T13:00:04,501 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T13:00:04,501 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): 065eb16105c4aedcbfa7fb1ec2b14c86/B is initiating minor compaction (all files) 2024-11-25T13:00:04,501 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 065eb16105c4aedcbfa7fb1ec2b14c86/B in TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:04,501 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/a85fb91fd49b47f3be87d55cc4ce7d09, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/b4882718154f4426abb1ed8dbe384cd9, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/b186d3dc5e1048e4a094684416935727] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp, totalSize=32.9 K 2024-11-25T13:00:04,501 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 84271 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T13:00:04,501 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1541): 065eb16105c4aedcbfa7fb1ec2b14c86/A is initiating minor compaction (all files) 2024-11-25T13:00:04,502 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 065eb16105c4aedcbfa7fb1ec2b14c86/A in TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:04,502 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/c8fa4d5e1cff4ce7af133036fcaa5c96, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/628f46655ebf45778131f9f8b76caf88, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/71f8c18f61bd4fab838ff17930b4a166] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp, totalSize=82.3 K 2024-11-25T13:00:04,502 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:04,502 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting a85fb91fd49b47f3be87d55cc4ce7d09, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=18, earliestPutTs=1732539601477 2024-11-25T13:00:04,502 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. files: [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/c8fa4d5e1cff4ce7af133036fcaa5c96, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/628f46655ebf45778131f9f8b76caf88, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/71f8c18f61bd4fab838ff17930b4a166] 2024-11-25T13:00:04,502 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting b4882718154f4426abb1ed8dbe384cd9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732539601486 2024-11-25T13:00:04,502 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8fa4d5e1cff4ce7af133036fcaa5c96, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=18, earliestPutTs=1732539601477 2024-11-25T13:00:04,502 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting b186d3dc5e1048e4a094684416935727, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732539602670 2024-11-25T13:00:04,503 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 628f46655ebf45778131f9f8b76caf88, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732539601486 2024-11-25T13:00:04,503 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71f8c18f61bd4fab838ff17930b4a166, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732539602670 2024-11-25T13:00:04,513 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 065eb16105c4aedcbfa7fb1ec2b14c86#B#compaction#473 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T13:00:04,514 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/0a8fd354251e449ea976e954aef6c896 is 50, key is test_row_0/B:col10/1732539602677/Put/seqid=0 2024-11-25T13:00:04,525 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T13:00:04,525 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 065eb16105c4aedcbfa7fb1ec2b14c86 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-25T13:00:04,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:04,527 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=065eb16105c4aedcbfa7fb1ec2b14c86] 2024-11-25T13:00:04,547 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241125d2de6831a2ce4110bc295e7f4afc6d91_065eb16105c4aedcbfa7fb1ec2b14c86 store=[table=TestAcidGuarantees family=A region=065eb16105c4aedcbfa7fb1ec2b14c86] 2024-11-25T13:00:04,549 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241125d2de6831a2ce4110bc295e7f4afc6d91_065eb16105c4aedcbfa7fb1ec2b14c86, store=[table=TestAcidGuarantees family=A region=065eb16105c4aedcbfa7fb1ec2b14c86] 2024-11-25T13:00:04,549 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125d2de6831a2ce4110bc295e7f4afc6d91_065eb16105c4aedcbfa7fb1ec2b14c86 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=065eb16105c4aedcbfa7fb1ec2b14c86] 2024-11-25T13:00:04,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742383_1559 (size=12104) 2024-11-25T13:00:04,584 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/0a8fd354251e449ea976e954aef6c896 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/0a8fd354251e449ea976e954aef6c896 2024-11-25T13:00:04,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742384_1560 (size=4469) 2024-11-25T13:00:04,592 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125a1f81cd22a0e48468bf67e9b5be3f06e_065eb16105c4aedcbfa7fb1ec2b14c86 is 50, key is test_row_0/A:col10/1732539603374/Put/seqid=0 2024-11-25T13:00:04,593 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 065eb16105c4aedcbfa7fb1ec2b14c86#A#compaction#474 average throughput is 0.37 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T13:00:04,593 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/06b361f16ce84fd2a4eaff947ed16b69 is 175, key is test_row_0/A:col10/1732539602677/Put/seqid=0 2024-11-25T13:00:04,597 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 065eb16105c4aedcbfa7fb1ec2b14c86/B of 065eb16105c4aedcbfa7fb1ec2b14c86 into 0a8fd354251e449ea976e954aef6c896(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T13:00:04,597 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 065eb16105c4aedcbfa7fb1ec2b14c86: 2024-11-25T13:00:04,597 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., storeName=065eb16105c4aedcbfa7fb1ec2b14c86/B, priority=13, startTime=1732539604500; duration=0sec 2024-11-25T13:00:04,597 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T13:00:04,597 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 065eb16105c4aedcbfa7fb1ec2b14c86:B 2024-11-25T13:00:04,597 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T13:00:04,598 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T13:00:04,598 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): 065eb16105c4aedcbfa7fb1ec2b14c86/C is initiating minor compaction (all files) 2024-11-25T13:00:04,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:04,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33764 deadline: 1732539664553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:04,599 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 065eb16105c4aedcbfa7fb1ec2b14c86/C in TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:04,599 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/9ed428103bca4174b11c33b713045b89, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/f8ff43fb7780480e8508e5edf99d76d2, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/897730c0e49945dda8098bafb3d628d8] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp, totalSize=32.9 K 2024-11-25T13:00:04,601 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ed428103bca4174b11c33b713045b89, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=18, earliestPutTs=1732539601477 2024-11-25T13:00:04,601 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting f8ff43fb7780480e8508e5edf99d76d2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732539601486 2024-11-25T13:00:04,602 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-25T13:00:04,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:04,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. as already flushing 2024-11-25T13:00:04,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:04,603 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:04,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:04,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:04,604 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:04,604 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:04,604 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:04,604 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 897730c0e49945dda8098bafb3d628d8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732539602670 2024-11-25T13:00:04,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:04,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33782 deadline: 1732539664598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:04,605 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:04,605 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:04,605 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:04,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:04,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1732539664605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:04,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:04,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33740 deadline: 1732539664605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:04,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:04,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:04,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:04,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:04,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:04,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:04,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742385_1561 (size=14594) 2024-11-25T13:00:04,620 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:04,625 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125a1f81cd22a0e48468bf67e9b5be3f06e_065eb16105c4aedcbfa7fb1ec2b14c86 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125a1f81cd22a0e48468bf67e9b5be3f06e_065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:04,626 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/f4c8294f23094e2688c5f02256bc3289, store: [table=TestAcidGuarantees family=A region=065eb16105c4aedcbfa7fb1ec2b14c86] 2024-11-25T13:00:04,626 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/f4c8294f23094e2688c5f02256bc3289 is 175, key is test_row_0/A:col10/1732539603374/Put/seqid=0 2024-11-25T13:00:04,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742386_1562 (size=31058) 2024-11-25T13:00:04,641 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 065eb16105c4aedcbfa7fb1ec2b14c86#C#compaction#476 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T13:00:04,641 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/b09f679fdb0049d79c24ee9928b7ebf5 is 50, key is test_row_0/C:col10/1732539602677/Put/seqid=0 2024-11-25T13:00:04,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742387_1563 (size=39549) 2024-11-25T13:00:04,671 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=77, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/f4c8294f23094e2688c5f02256bc3289 2024-11-25T13:00:04,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742388_1564 (size=12104) 2024-11-25T13:00:04,713 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/f202eddfeafb4fddbfc6c93379e91569 is 50, key is test_row_0/B:col10/1732539603374/Put/seqid=0 2024-11-25T13:00:04,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:04,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33764 deadline: 1732539664710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:04,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:04,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33782 deadline: 1732539664711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:04,714 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:04,714 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:04,714 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:04,714 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:04,714 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:04,714 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:04,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:04,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33740 deadline: 1732539664720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:04,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:04,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1732539664720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:04,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:04,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:04,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:04,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:04,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:04,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:04,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742389_1565 (size=12001) 2024-11-25T13:00:04,733 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/f202eddfeafb4fddbfc6c93379e91569 2024-11-25T13:00:04,742 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/2a60d1c95d444d7caddcd5f7b7358050 is 50, key is test_row_0/C:col10/1732539603374/Put/seqid=0 2024-11-25T13:00:04,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742390_1566 (size=12001) 2024-11-25T13:00:04,750 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/2a60d1c95d444d7caddcd5f7b7358050 2024-11-25T13:00:04,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/f4c8294f23094e2688c5f02256bc3289 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/f4c8294f23094e2688c5f02256bc3289 2024-11-25T13:00:04,756 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-25T13:00:04,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:04,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. as already flushing 2024-11-25T13:00:04,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:04,757 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:04,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:04,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:04,760 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/f4c8294f23094e2688c5f02256bc3289, entries=200, sequenceid=77, filesize=38.6 K 2024-11-25T13:00:04,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/f202eddfeafb4fddbfc6c93379e91569 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/f202eddfeafb4fddbfc6c93379e91569 2024-11-25T13:00:04,766 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/f202eddfeafb4fddbfc6c93379e91569, entries=150, sequenceid=77, filesize=11.7 K 2024-11-25T13:00:04,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/2a60d1c95d444d7caddcd5f7b7358050 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/2a60d1c95d444d7caddcd5f7b7358050 2024-11-25T13:00:04,771 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/2a60d1c95d444d7caddcd5f7b7358050, entries=150, sequenceid=77, filesize=11.7 K 2024-11-25T13:00:04,772 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 065eb16105c4aedcbfa7fb1ec2b14c86 in 246ms, sequenceid=77, compaction requested=false 2024-11-25T13:00:04,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 065eb16105c4aedcbfa7fb1ec2b14c86: 2024-11-25T13:00:04,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-25T13:00:04,909 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-25T13:00:04,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:04,909 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2902): Flushing 065eb16105c4aedcbfa7fb1ec2b14c86 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-25T13:00:04,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:04,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. as already flushing 2024-11-25T13:00:04,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125cef94469a5754d7b8d6efd9010cdeb47_065eb16105c4aedcbfa7fb1ec2b14c86 is 50, key is test_row_0/A:col10/1732539604543/Put/seqid=0 2024-11-25T13:00:04,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742391_1567 (size=12154) 2024-11-25T13:00:04,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:04,948 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125cef94469a5754d7b8d6efd9010cdeb47_065eb16105c4aedcbfa7fb1ec2b14c86 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125cef94469a5754d7b8d6efd9010cdeb47_065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:04,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/d16f399185e640bcab04b23f289866e5, store: [table=TestAcidGuarantees family=A region=065eb16105c4aedcbfa7fb1ec2b14c86] 2024-11-25T13:00:04,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/d16f399185e640bcab04b23f289866e5 is 175, key is test_row_0/A:col10/1732539604543/Put/seqid=0 2024-11-25T13:00:04,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742392_1568 (size=30955) 2024-11-25T13:00:04,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:04,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33782 deadline: 1732539664970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:04,982 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:04,982 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:04,982 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:04,983 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=92, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/d16f399185e640bcab04b23f289866e5 2024-11-25T13:00:04,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:04,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33764 deadline: 1732539664982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:04,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:04,990 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:04,990 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:04,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33740 deadline: 1732539664983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:04,990 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:04,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:04,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:04,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:04,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:04,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1732539664984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:04,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:04,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:04,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:05,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/bf5387e09c534cea819c5380096bb271 is 50, key is test_row_0/B:col10/1732539604543/Put/seqid=0 2024-11-25T13:00:05,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742393_1569 (size=12001) 2024-11-25T13:00:05,039 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/06b361f16ce84fd2a4eaff947ed16b69 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/06b361f16ce84fd2a4eaff947ed16b69 2024-11-25T13:00:05,044 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 065eb16105c4aedcbfa7fb1ec2b14c86/A of 065eb16105c4aedcbfa7fb1ec2b14c86 into 06b361f16ce84fd2a4eaff947ed16b69(size=30.3 K), total size for store is 69.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T13:00:05,044 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 065eb16105c4aedcbfa7fb1ec2b14c86: 2024-11-25T13:00:05,044 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., storeName=065eb16105c4aedcbfa7fb1ec2b14c86/A, priority=13, startTime=1732539604500; duration=0sec 2024-11-25T13:00:05,044 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T13:00:05,044 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 065eb16105c4aedcbfa7fb1ec2b14c86:A 2024-11-25T13:00:05,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:05,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33782 deadline: 1732539665090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:05,097 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:05,097 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:05,097 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:05,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:05,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33740 deadline: 1732539665100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:05,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:05,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:05,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33764 deadline: 1732539665100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:05,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:05,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:05,103 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:05,103 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:05,103 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:05,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:05,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1732539665100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:05,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:05,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:05,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:05,110 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/b09f679fdb0049d79c24ee9928b7ebf5 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/b09f679fdb0049d79c24ee9928b7ebf5 2024-11-25T13:00:05,123 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 065eb16105c4aedcbfa7fb1ec2b14c86/C of 065eb16105c4aedcbfa7fb1ec2b14c86 into b09f679fdb0049d79c24ee9928b7ebf5(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T13:00:05,123 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 065eb16105c4aedcbfa7fb1ec2b14c86: 2024-11-25T13:00:05,124 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., storeName=065eb16105c4aedcbfa7fb1ec2b14c86/C, priority=13, startTime=1732539604500; duration=0sec 2024-11-25T13:00:05,124 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T13:00:05,124 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 065eb16105c4aedcbfa7fb1ec2b14c86:C 2024-11-25T13:00:05,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:05,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33782 deadline: 1732539665300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:05,303 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:05,303 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:05,303 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:05,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:05,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33740 deadline: 1732539665310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:05,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:05,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:05,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:05,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:05,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1732539665310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:05,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:05,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33764 deadline: 1732539665310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:05,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:05,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:05,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:05,314 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:05,314 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:05,314 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:05,439 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/bf5387e09c534cea819c5380096bb271 2024-11-25T13:00:05,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/a9a1e5503d374888ab55d8b96206586d is 50, key is test_row_0/C:col10/1732539604543/Put/seqid=0 2024-11-25T13:00:05,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742394_1570 (size=12001) 2024-11-25T13:00:05,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:05,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33782 deadline: 1732539665612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:05,619 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:05,619 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:05,619 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:05,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:05,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1732539665622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:05,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:05,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:05,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:05,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:05,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33764 deadline: 1732539665622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:05,628 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:05,628 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:05,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:05,628 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:05,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33740 deadline: 1732539665622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:05,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:05,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:05,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:05,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:05,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33778 deadline: 1732539665722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:05,728 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 1ff8009e463e,35905,1732539449246 for 'test_row_2' in 065eb16105c4aedcbfa7fb1ec2b14c86 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T13:00:05,729 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:05,729 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:05,729 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:05,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-25T13:00:05,932 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/a9a1e5503d374888ab55d8b96206586d 2024-11-25T13:00:05,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/d16f399185e640bcab04b23f289866e5 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/d16f399185e640bcab04b23f289866e5 2024-11-25T13:00:05,967 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/d16f399185e640bcab04b23f289866e5, entries=150, sequenceid=92, filesize=30.2 K 2024-11-25T13:00:05,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/bf5387e09c534cea819c5380096bb271 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/bf5387e09c534cea819c5380096bb271 2024-11-25T13:00:05,981 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/bf5387e09c534cea819c5380096bb271, entries=150, sequenceid=92, filesize=11.7 K 2024-11-25T13:00:05,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/a9a1e5503d374888ab55d8b96206586d as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/a9a1e5503d374888ab55d8b96206586d 2024-11-25T13:00:06,000 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/a9a1e5503d374888ab55d8b96206586d, entries=150, sequenceid=92, filesize=11.7 K 2024-11-25T13:00:06,001 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for 065eb16105c4aedcbfa7fb1ec2b14c86 in 1092ms, sequenceid=92, compaction requested=true 2024-11-25T13:00:06,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2603): Flush status journal for 065eb16105c4aedcbfa7fb1ec2b14c86: 2024-11-25T13:00:06,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:06,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=143 2024-11-25T13:00:06,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4169): Remote procedure done, pid=143 2024-11-25T13:00:06,005 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-25T13:00:06,005 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3450 sec 2024-11-25T13:00:06,007 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees in 2.3510 sec 2024-11-25T13:00:06,139 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T13:00:06,139 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 065eb16105c4aedcbfa7fb1ec2b14c86 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-25T13:00:06,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:06,170 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411255c98f39e253f4fcf8836e76b7d84340e_065eb16105c4aedcbfa7fb1ec2b14c86 is 50, key is test_row_0/A:col10/1732539604978/Put/seqid=0 2024-11-25T13:00:06,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742395_1571 (size=17034) 2024-11-25T13:00:06,188 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:06,191 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411255c98f39e253f4fcf8836e76b7d84340e_065eb16105c4aedcbfa7fb1ec2b14c86 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411255c98f39e253f4fcf8836e76b7d84340e_065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:06,193 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/5fbd81d77b744594a5c4bb38d52feb8b, store: [table=TestAcidGuarantees family=A region=065eb16105c4aedcbfa7fb1ec2b14c86] 2024-11-25T13:00:06,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:06,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33740 deadline: 1732539666184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:06,193 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/5fbd81d77b744594a5c4bb38d52feb8b is 175, key is test_row_0/A:col10/1732539604978/Put/seqid=0 2024-11-25T13:00:06,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:06,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:06,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:06,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:06,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33764 deadline: 1732539666196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:06,211 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:06,211 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:06,211 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:06,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:06,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33782 deadline: 1732539666204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:06,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:06,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1732539666208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:06,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:06,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:06,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:06,215 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:06,215 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:06,215 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:06,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742396_1572 (size=48139) 2024-11-25T13:00:06,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:06,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33740 deadline: 1732539666301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:06,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:06,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:06,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:06,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:06,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33764 deadline: 1732539666319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:06,326 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:06,326 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:06,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:06,326 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:06,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1732539666320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:06,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:06,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:06,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:06,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33782 deadline: 1732539666320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:06,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:06,327 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:06,327 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:06,327 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:06,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:06,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33740 deadline: 1732539666520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:06,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:06,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:06,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:06,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:06,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33764 deadline: 1732539666530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:06,533 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:06,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:06,533 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:06,533 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:06,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1732539666530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:06,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:06,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:06,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:06,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:06,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33782 deadline: 1732539666530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:06,534 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:06,534 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:06,534 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:06,626 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=119, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/5fbd81d77b744594a5c4bb38d52feb8b 2024-11-25T13:00:06,646 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/28af56c923014440b0e0f9781864d8aa is 50, key is test_row_0/B:col10/1732539604978/Put/seqid=0 2024-11-25T13:00:06,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742397_1573 (size=12001) 2024-11-25T13:00:06,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:06,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33740 deadline: 1732539666830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:06,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:06,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:06,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:06,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:06,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33764 deadline: 1732539666843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:06,845 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:06,845 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:06,845 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:06,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:06,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33782 deadline: 1732539666843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:06,847 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:06,847 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:06,847 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:06,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:06,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1732539666844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:06,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:06,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:06,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:07,109 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/28af56c923014440b0e0f9781864d8aa 2024-11-25T13:00:07,128 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/6257f9db72f743d8ae7474c663c0b0c7 is 50, key is test_row_0/C:col10/1732539604978/Put/seqid=0 2024-11-25T13:00:07,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742398_1574 (size=12001) 2024-11-25T13:00:07,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:07,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33740 deadline: 1732539667340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:07,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:07,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:07,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:07,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:07,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33764 deadline: 1732539667350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:07,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:07,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33782 deadline: 1732539667350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:07,358 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:07,358 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:07,358 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:07,358 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:07,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:07,359 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:07,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:07,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1732539667361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:07,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:07,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:07,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:07,575 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/6257f9db72f743d8ae7474c663c0b0c7 2024-11-25T13:00:07,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/5fbd81d77b744594a5c4bb38d52feb8b as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/5fbd81d77b744594a5c4bb38d52feb8b 2024-11-25T13:00:07,606 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/5fbd81d77b744594a5c4bb38d52feb8b, entries=250, sequenceid=119, filesize=47.0 K 2024-11-25T13:00:07,619 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/28af56c923014440b0e0f9781864d8aa as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/28af56c923014440b0e0f9781864d8aa 2024-11-25T13:00:07,656 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/28af56c923014440b0e0f9781864d8aa, entries=150, sequenceid=119, filesize=11.7 K 2024-11-25T13:00:07,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/6257f9db72f743d8ae7474c663c0b0c7 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/6257f9db72f743d8ae7474c663c0b0c7 2024-11-25T13:00:07,661 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/6257f9db72f743d8ae7474c663c0b0c7, entries=150, sequenceid=119, filesize=11.7 K 2024-11-25T13:00:07,662 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=67.09 KB/68700 for 065eb16105c4aedcbfa7fb1ec2b14c86 in 1523ms, sequenceid=119, compaction requested=true 2024-11-25T13:00:07,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 065eb16105c4aedcbfa7fb1ec2b14c86: 2024-11-25T13:00:07,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 065eb16105c4aedcbfa7fb1ec2b14c86:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T13:00:07,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T13:00:07,662 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T13:00:07,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 065eb16105c4aedcbfa7fb1ec2b14c86:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T13:00:07,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T13:00:07,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 065eb16105c4aedcbfa7fb1ec2b14c86:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T13:00:07,662 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T13:00:07,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T13:00:07,664 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T13:00:07,664 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): 065eb16105c4aedcbfa7fb1ec2b14c86/B is initiating minor compaction (all files) 2024-11-25T13:00:07,664 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 065eb16105c4aedcbfa7fb1ec2b14c86/B in TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:07,664 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/0a8fd354251e449ea976e954aef6c896, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/f202eddfeafb4fddbfc6c93379e91569, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/bf5387e09c534cea819c5380096bb271, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/28af56c923014440b0e0f9781864d8aa] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp, totalSize=47.0 K 2024-11-25T13:00:07,664 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 149701 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T13:00:07,664 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1541): 065eb16105c4aedcbfa7fb1ec2b14c86/A is initiating minor compaction (all files) 2024-11-25T13:00:07,664 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 065eb16105c4aedcbfa7fb1ec2b14c86/A in TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:07,664 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/06b361f16ce84fd2a4eaff947ed16b69, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/f4c8294f23094e2688c5f02256bc3289, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/d16f399185e640bcab04b23f289866e5, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/5fbd81d77b744594a5c4bb38d52feb8b] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp, totalSize=146.2 K 2024-11-25T13:00:07,664 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:07,664 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. files: [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/06b361f16ce84fd2a4eaff947ed16b69, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/f4c8294f23094e2688c5f02256bc3289, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/d16f399185e640bcab04b23f289866e5, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/5fbd81d77b744594a5c4bb38d52feb8b] 2024-11-25T13:00:07,665 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06b361f16ce84fd2a4eaff947ed16b69, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732539602670 2024-11-25T13:00:07,665 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a8fd354251e449ea976e954aef6c896, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732539602670 2024-11-25T13:00:07,666 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4c8294f23094e2688c5f02256bc3289, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732539603364 2024-11-25T13:00:07,666 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting f202eddfeafb4fddbfc6c93379e91569, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732539603373 2024-11-25T13:00:07,666 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting d16f399185e640bcab04b23f289866e5, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732539604542 2024-11-25T13:00:07,667 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting bf5387e09c534cea819c5380096bb271, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732539604542 2024-11-25T13:00:07,667 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5fbd81d77b744594a5c4bb38d52feb8b, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732539604948 2024-11-25T13:00:07,667 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 28af56c923014440b0e0f9781864d8aa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732539604948 2024-11-25T13:00:07,680 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 065eb16105c4aedcbfa7fb1ec2b14c86#B#compaction#485 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T13:00:07,680 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/e9f7da0583744882a2853c1fe7c9502d is 50, key is test_row_0/B:col10/1732539604978/Put/seqid=0 2024-11-25T13:00:07,696 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=065eb16105c4aedcbfa7fb1ec2b14c86] 2024-11-25T13:00:07,713 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112517be87fc454148b38eedd238bb6d3b25_065eb16105c4aedcbfa7fb1ec2b14c86 store=[table=TestAcidGuarantees family=A region=065eb16105c4aedcbfa7fb1ec2b14c86] 2024-11-25T13:00:07,715 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112517be87fc454148b38eedd238bb6d3b25_065eb16105c4aedcbfa7fb1ec2b14c86, store=[table=TestAcidGuarantees family=A region=065eb16105c4aedcbfa7fb1ec2b14c86] 2024-11-25T13:00:07,715 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112517be87fc454148b38eedd238bb6d3b25_065eb16105c4aedcbfa7fb1ec2b14c86 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=065eb16105c4aedcbfa7fb1ec2b14c86] 2024-11-25T13:00:07,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742399_1575 (size=12241) 2024-11-25T13:00:07,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742400_1576 (size=4469) 2024-11-25T13:00:07,784 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 065eb16105c4aedcbfa7fb1ec2b14c86#A#compaction#486 average throughput is 0.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T13:00:07,785 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/917b97f9ed504fc880b8f6529317ffb8 is 175, key is test_row_0/A:col10/1732539604978/Put/seqid=0 2024-11-25T13:00:07,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-25T13:00:07,820 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T13:00:07,821 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T13:00:07,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] procedure2.ProcedureExecutor(1139): Stored pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees 2024-11-25T13:00:07,824 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T13:00:07,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-25T13:00:07,825 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T13:00:07,825 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T13:00:07,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742401_1577 (size=31195) 2024-11-25T13:00:07,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-25T13:00:07,979 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-25T13:00:07,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:07,979 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2902): Flushing 065eb16105c4aedcbfa7fb1ec2b14c86 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-25T13:00:08,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125e59dc0d1c1594bbfae08f02280d14d62_065eb16105c4aedcbfa7fb1ec2b14c86 is 50, key is test_row_0/A:col10/1732539606180/Put/seqid=0 2024-11-25T13:00:08,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742402_1578 (size=12254) 2024-11-25T13:00:08,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:08,111 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125e59dc0d1c1594bbfae08f02280d14d62_065eb16105c4aedcbfa7fb1ec2b14c86 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125e59dc0d1c1594bbfae08f02280d14d62_065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:08,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/dc9d76ac80a64714a17964d357ff8e4d, store: [table=TestAcidGuarantees family=A region=065eb16105c4aedcbfa7fb1ec2b14c86] 2024-11-25T13:00:08,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/dc9d76ac80a64714a17964d357ff8e4d is 175, key is test_row_0/A:col10/1732539606180/Put/seqid=0 2024-11-25T13:00:08,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-25T13:00:08,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742403_1579 (size=31055) 2024-11-25T13:00:08,163 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/e9f7da0583744882a2853c1fe7c9502d as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/e9f7da0583744882a2853c1fe7c9502d 2024-11-25T13:00:08,173 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 065eb16105c4aedcbfa7fb1ec2b14c86/B of 065eb16105c4aedcbfa7fb1ec2b14c86 into e9f7da0583744882a2853c1fe7c9502d(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T13:00:08,173 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 065eb16105c4aedcbfa7fb1ec2b14c86: 2024-11-25T13:00:08,173 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., storeName=065eb16105c4aedcbfa7fb1ec2b14c86/B, priority=12, startTime=1732539607662; duration=0sec 2024-11-25T13:00:08,173 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T13:00:08,173 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 065eb16105c4aedcbfa7fb1ec2b14c86:B 2024-11-25T13:00:08,173 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T13:00:08,174 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T13:00:08,174 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): 065eb16105c4aedcbfa7fb1ec2b14c86/C is initiating minor compaction (all files) 2024-11-25T13:00:08,175 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 065eb16105c4aedcbfa7fb1ec2b14c86/C in TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:08,175 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/b09f679fdb0049d79c24ee9928b7ebf5, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/2a60d1c95d444d7caddcd5f7b7358050, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/a9a1e5503d374888ab55d8b96206586d, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/6257f9db72f743d8ae7474c663c0b0c7] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp, totalSize=47.0 K 2024-11-25T13:00:08,175 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting b09f679fdb0049d79c24ee9928b7ebf5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732539602670 2024-11-25T13:00:08,176 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a60d1c95d444d7caddcd5f7b7358050, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732539603373 2024-11-25T13:00:08,176 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting a9a1e5503d374888ab55d8b96206586d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732539604542 2024-11-25T13:00:08,176 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 6257f9db72f743d8ae7474c663c0b0c7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732539604948 2024-11-25T13:00:08,194 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 065eb16105c4aedcbfa7fb1ec2b14c86#C#compaction#488 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T13:00:08,195 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/79cc8d29a6c54f48889a051eeaafe0b0 is 50, key is test_row_0/C:col10/1732539604978/Put/seqid=0 2024-11-25T13:00:08,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742404_1580 (size=12241) 2024-11-25T13:00:08,248 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/917b97f9ed504fc880b8f6529317ffb8 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/917b97f9ed504fc880b8f6529317ffb8 2024-11-25T13:00:08,257 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 065eb16105c4aedcbfa7fb1ec2b14c86/A of 065eb16105c4aedcbfa7fb1ec2b14c86 into 917b97f9ed504fc880b8f6529317ffb8(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T13:00:08,257 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 065eb16105c4aedcbfa7fb1ec2b14c86: 2024-11-25T13:00:08,257 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., storeName=065eb16105c4aedcbfa7fb1ec2b14c86/A, priority=12, startTime=1732539607662; duration=0sec 2024-11-25T13:00:08,257 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T13:00:08,257 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 065eb16105c4aedcbfa7fb1ec2b14c86:A 2024-11-25T13:00:08,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:08,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. as already flushing 2024-11-25T13:00:08,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:08,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33740 deadline: 1732539668421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:08,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:08,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:08,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:08,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:08,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1732539668426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:08,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:08,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:08,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:08,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:08,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33764 deadline: 1732539668427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:08,432 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:08,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:08,432 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:08,432 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:08,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33782 deadline: 1732539668427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:08,435 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:08,435 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:08,435 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:08,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-25T13:00:08,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:08,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33740 deadline: 1732539668531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:08,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:08,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:08,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:08,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:08,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1732539668543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:08,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:08,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33764 deadline: 1732539668544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:08,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:08,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:08,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:08,548 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:08,548 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:08,548 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:08,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:08,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33782 deadline: 1732539668544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:08,549 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:08,549 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:08,549 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:08,560 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=132, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/dc9d76ac80a64714a17964d357ff8e4d 2024-11-25T13:00:08,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/b9f126dfaa1842f3aea1d44697469793 is 50, key is test_row_0/B:col10/1732539606180/Put/seqid=0 2024-11-25T13:00:08,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742405_1581 (size=12101) 2024-11-25T13:00:08,638 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/b9f126dfaa1842f3aea1d44697469793 2024-11-25T13:00:08,640 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/79cc8d29a6c54f48889a051eeaafe0b0 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/79cc8d29a6c54f48889a051eeaafe0b0 2024-11-25T13:00:08,649 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 065eb16105c4aedcbfa7fb1ec2b14c86/C of 065eb16105c4aedcbfa7fb1ec2b14c86 into 79cc8d29a6c54f48889a051eeaafe0b0(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T13:00:08,650 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 065eb16105c4aedcbfa7fb1ec2b14c86: 2024-11-25T13:00:08,650 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., storeName=065eb16105c4aedcbfa7fb1ec2b14c86/C, priority=12, startTime=1732539607662; duration=0sec 2024-11-25T13:00:08,650 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T13:00:08,650 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 065eb16105c4aedcbfa7fb1ec2b14c86:C 2024-11-25T13:00:08,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/6885c0ca502d47d995bb0d8948f8b9f5 is 50, key is test_row_0/C:col10/1732539606180/Put/seqid=0 2024-11-25T13:00:08,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742406_1582 (size=12101) 2024-11-25T13:00:08,697 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/6885c0ca502d47d995bb0d8948f8b9f5 2024-11-25T13:00:08,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/dc9d76ac80a64714a17964d357ff8e4d as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/dc9d76ac80a64714a17964d357ff8e4d 2024-11-25T13:00:08,734 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/dc9d76ac80a64714a17964d357ff8e4d, entries=150, sequenceid=132, filesize=30.3 K 2024-11-25T13:00:08,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/b9f126dfaa1842f3aea1d44697469793 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/b9f126dfaa1842f3aea1d44697469793 2024-11-25T13:00:08,748 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/b9f126dfaa1842f3aea1d44697469793, entries=150, sequenceid=132, filesize=11.8 K 2024-11-25T13:00:08,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:08,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33740 deadline: 1732539668743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:08,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/6885c0ca502d47d995bb0d8948f8b9f5 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/6885c0ca502d47d995bb0d8948f8b9f5 2024-11-25T13:00:08,753 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/6885c0ca502d47d995bb0d8948f8b9f5, entries=150, sequenceid=132, filesize=11.8 K 2024-11-25T13:00:08,754 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 065eb16105c4aedcbfa7fb1ec2b14c86 in 775ms, sequenceid=132, compaction requested=false 2024-11-25T13:00:08,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2603): Flush status journal for 065eb16105c4aedcbfa7fb1ec2b14c86: 2024-11-25T13:00:08,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:08,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=145 2024-11-25T13:00:08,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4169): Remote procedure done, pid=145 2024-11-25T13:00:08,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:08,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:08,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:08,758 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-25T13:00:08,758 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 931 msec 2024-11-25T13:00:08,760 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees in 936 msec 2024-11-25T13:00:08,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:08,761 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T13:00:08,761 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 065eb16105c4aedcbfa7fb1ec2b14c86 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-25T13:00:08,779 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125f3421c23089148c69d27092ac8490b81_065eb16105c4aedcbfa7fb1ec2b14c86 is 50, key is test_row_0/A:col10/1732539608425/Put/seqid=0 2024-11-25T13:00:08,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:08,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1732539668784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:08,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:08,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:08,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:08,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:08,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33764 deadline: 1732539668788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:08,793 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:08,793 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:08,793 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:08,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:08,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33782 deadline: 1732539668791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:08,796 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:08,796 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:08,796 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:08,811 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-25T13:00:08,811 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-25T13:00:08,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742407_1583 (size=14794) 2024-11-25T13:00:08,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:08,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1732539668901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:08,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:08,906 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:08,906 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:08,906 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:08,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33764 deadline: 1732539668901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:08,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:08,906 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:08,906 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:08,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33782 deadline: 1732539668901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:08,906 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:08,906 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:08,906 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:08,906 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:08,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-25T13:00:08,960 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T13:00:08,960 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T13:00:08,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees 2024-11-25T13:00:08,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-25T13:00:08,962 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T13:00:08,964 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T13:00:08,964 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T13:00:09,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:09,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33740 deadline: 1732539669060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:09,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:09,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:09,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:09,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-25T13:00:09,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:09,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1732539669111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:09,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:09,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:09,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:09,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:09,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33764 deadline: 1732539669111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:09,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:09,117 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:09,117 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:09,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33782 deadline: 1732539669111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:09,117 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:09,117 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:09,117 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:09,117 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:09,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-25T13:00:09,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:09,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. as already flushing 2024-11-25T13:00:09,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:09,120 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:09,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:09,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:09,218 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:09,239 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125f3421c23089148c69d27092ac8490b81_065eb16105c4aedcbfa7fb1ec2b14c86 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125f3421c23089148c69d27092ac8490b81_065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:09,241 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/6bde8b420bb949e7b3b1278988b2163f, store: [table=TestAcidGuarantees family=A region=065eb16105c4aedcbfa7fb1ec2b14c86] 2024-11-25T13:00:09,241 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/6bde8b420bb949e7b3b1278988b2163f is 175, key is test_row_0/A:col10/1732539608425/Put/seqid=0 2024-11-25T13:00:09,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742408_1584 (size=39749) 2024-11-25T13:00:09,271 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=160, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/6bde8b420bb949e7b3b1278988b2163f 2024-11-25T13:00:09,274 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-25T13:00:09,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:09,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. as already flushing 2024-11-25T13:00:09,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:09,274 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:09,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:09,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:09,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-25T13:00:09,286 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/7268c14c7213414182cbcc59bf30a206 is 50, key is test_row_0/B:col10/1732539608425/Put/seqid=0 2024-11-25T13:00:09,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742409_1585 (size=12151) 2024-11-25T13:00:09,346 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/7268c14c7213414182cbcc59bf30a206 2024-11-25T13:00:09,353 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/e54c84cf8c374916b5d6622fe1a18c68 is 50, key is test_row_0/C:col10/1732539608425/Put/seqid=0 2024-11-25T13:00:09,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742410_1586 (size=12151) 2024-11-25T13:00:09,375 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/e54c84cf8c374916b5d6622fe1a18c68 2024-11-25T13:00:09,380 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/6bde8b420bb949e7b3b1278988b2163f as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/6bde8b420bb949e7b3b1278988b2163f 2024-11-25T13:00:09,386 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/6bde8b420bb949e7b3b1278988b2163f, entries=200, sequenceid=160, filesize=38.8 K 2024-11-25T13:00:09,387 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/7268c14c7213414182cbcc59bf30a206 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/7268c14c7213414182cbcc59bf30a206 2024-11-25T13:00:09,390 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/7268c14c7213414182cbcc59bf30a206, entries=150, sequenceid=160, filesize=11.9 K 2024-11-25T13:00:09,390 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/e54c84cf8c374916b5d6622fe1a18c68 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/e54c84cf8c374916b5d6622fe1a18c68 2024-11-25T13:00:09,406 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/e54c84cf8c374916b5d6622fe1a18c68, entries=150, sequenceid=160, filesize=11.9 K 2024-11-25T13:00:09,415 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 065eb16105c4aedcbfa7fb1ec2b14c86 in 654ms, sequenceid=160, compaction requested=true 2024-11-25T13:00:09,415 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 065eb16105c4aedcbfa7fb1ec2b14c86: 2024-11-25T13:00:09,415 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T13:00:09,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 065eb16105c4aedcbfa7fb1ec2b14c86:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T13:00:09,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T13:00:09,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 065eb16105c4aedcbfa7fb1ec2b14c86:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T13:00:09,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T13:00:09,420 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T13:00:09,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 065eb16105c4aedcbfa7fb1ec2b14c86:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T13:00:09,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T13:00:09,424 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101999 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T13:00:09,424 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1541): 065eb16105c4aedcbfa7fb1ec2b14c86/A is initiating minor compaction (all files) 2024-11-25T13:00:09,424 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 065eb16105c4aedcbfa7fb1ec2b14c86/A in TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:09,425 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/917b97f9ed504fc880b8f6529317ffb8, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/dc9d76ac80a64714a17964d357ff8e4d, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/6bde8b420bb949e7b3b1278988b2163f] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp, totalSize=99.6 K 2024-11-25T13:00:09,425 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:09,425 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. files: [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/917b97f9ed504fc880b8f6529317ffb8, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/dc9d76ac80a64714a17964d357ff8e4d, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/6bde8b420bb949e7b3b1278988b2163f] 2024-11-25T13:00:09,427 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 917b97f9ed504fc880b8f6529317ffb8, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732539604948 2024-11-25T13:00:09,427 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36493 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T13:00:09,427 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): 065eb16105c4aedcbfa7fb1ec2b14c86/B is initiating minor compaction (all files) 2024-11-25T13:00:09,427 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 065eb16105c4aedcbfa7fb1ec2b14c86/B in TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:09,427 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/e9f7da0583744882a2853c1fe7c9502d, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/b9f126dfaa1842f3aea1d44697469793, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/7268c14c7213414182cbcc59bf30a206] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp, totalSize=35.6 K 2024-11-25T13:00:09,427 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc9d76ac80a64714a17964d357ff8e4d, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732539606180 2024-11-25T13:00:09,428 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting e9f7da0583744882a2853c1fe7c9502d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732539604948 2024-11-25T13:00:09,431 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6bde8b420bb949e7b3b1278988b2163f, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1732539608405 2024-11-25T13:00:09,432 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting b9f126dfaa1842f3aea1d44697469793, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732539606180 2024-11-25T13:00:09,435 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 7268c14c7213414182cbcc59bf30a206, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1732539608424 2024-11-25T13:00:09,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:09,435 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-25T13:00:09,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:09,436 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2902): Flushing 065eb16105c4aedcbfa7fb1ec2b14c86 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-25T13:00:09,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. as already flushing 2024-11-25T13:00:09,456 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=065eb16105c4aedcbfa7fb1ec2b14c86] 2024-11-25T13:00:09,468 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 065eb16105c4aedcbfa7fb1ec2b14c86#B#compaction#495 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T13:00:09,468 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/65d601588e17457cb335f172d7d1832f is 50, key is test_row_0/B:col10/1732539608425/Put/seqid=0 2024-11-25T13:00:09,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125aeb01705bb1c4151b7f0e0f3afc17c34_065eb16105c4aedcbfa7fb1ec2b14c86 is 50, key is test_row_0/A:col10/1732539609429/Put/seqid=0 2024-11-25T13:00:09,477 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112521024ddedeb143a3a02569703c0bafaf_065eb16105c4aedcbfa7fb1ec2b14c86 store=[table=TestAcidGuarantees family=A region=065eb16105c4aedcbfa7fb1ec2b14c86] 2024-11-25T13:00:09,484 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112521024ddedeb143a3a02569703c0bafaf_065eb16105c4aedcbfa7fb1ec2b14c86, store=[table=TestAcidGuarantees family=A region=065eb16105c4aedcbfa7fb1ec2b14c86] 2024-11-25T13:00:09,484 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112521024ddedeb143a3a02569703c0bafaf_065eb16105c4aedcbfa7fb1ec2b14c86 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=065eb16105c4aedcbfa7fb1ec2b14c86] 2024-11-25T13:00:09,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742411_1587 (size=12493) 2024-11-25T13:00:09,544 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/65d601588e17457cb335f172d7d1832f as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/65d601588e17457cb335f172d7d1832f 2024-11-25T13:00:09,553 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 065eb16105c4aedcbfa7fb1ec2b14c86/B of 065eb16105c4aedcbfa7fb1ec2b14c86 into 65d601588e17457cb335f172d7d1832f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T13:00:09,553 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 065eb16105c4aedcbfa7fb1ec2b14c86: 2024-11-25T13:00:09,553 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., storeName=065eb16105c4aedcbfa7fb1ec2b14c86/B, priority=13, startTime=1732539609420; duration=0sec 2024-11-25T13:00:09,553 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T13:00:09,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742412_1588 (size=17284) 2024-11-25T13:00:09,554 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 065eb16105c4aedcbfa7fb1ec2b14c86:B 2024-11-25T13:00:09,554 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T13:00:09,556 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36493 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T13:00:09,556 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): 065eb16105c4aedcbfa7fb1ec2b14c86/C is initiating minor compaction (all files) 2024-11-25T13:00:09,556 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 065eb16105c4aedcbfa7fb1ec2b14c86/C in TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:09,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:09,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33764 deadline: 1732539669556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:09,557 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/79cc8d29a6c54f48889a051eeaafe0b0, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/6885c0ca502d47d995bb0d8948f8b9f5, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/e54c84cf8c374916b5d6622fe1a18c68] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp, totalSize=35.6 K 2024-11-25T13:00:09,557 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 79cc8d29a6c54f48889a051eeaafe0b0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732539604948 2024-11-25T13:00:09,557 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 6885c0ca502d47d995bb0d8948f8b9f5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732539606180 2024-11-25T13:00:09,558 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting e54c84cf8c374916b5d6622fe1a18c68, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1732539608424 2024-11-25T13:00:09,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:09,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:09,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:09,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742413_1589 (size=4469) 2024-11-25T13:00:09,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:09,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1732539669579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:09,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:09,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33782 deadline: 1732539669579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:09,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:09,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33740 deadline: 1732539669581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:09,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:09,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:09,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:09,588 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:09,588 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:09,588 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:09,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-25T13:00:09,592 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 065eb16105c4aedcbfa7fb1ec2b14c86#C#compaction#497 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T13:00:09,593 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/ac11773566104d58ac71b3384593cc57 is 50, key is test_row_0/C:col10/1732539608425/Put/seqid=0 2024-11-25T13:00:09,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:09,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:09,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:09,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742414_1590 (size=12493) 2024-11-25T13:00:09,652 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/ac11773566104d58ac71b3384593cc57 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/ac11773566104d58ac71b3384593cc57 2024-11-25T13:00:09,656 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 065eb16105c4aedcbfa7fb1ec2b14c86/C of 065eb16105c4aedcbfa7fb1ec2b14c86 into ac11773566104d58ac71b3384593cc57(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T13:00:09,656 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 065eb16105c4aedcbfa7fb1ec2b14c86: 2024-11-25T13:00:09,656 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., storeName=065eb16105c4aedcbfa7fb1ec2b14c86/C, priority=13, startTime=1732539609421; duration=0sec 2024-11-25T13:00:09,656 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T13:00:09,656 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 065eb16105c4aedcbfa7fb1ec2b14c86:C 2024-11-25T13:00:09,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:09,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33764 deadline: 1732539669671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:09,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:09,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:09,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:09,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:09,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1732539669690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:09,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:09,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:09,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:09,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:09,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33782 deadline: 1732539669690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:09,704 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:09,704 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:09,704 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:09,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:09,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33778 deadline: 1732539669771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:09,775 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 1ff8009e463e,35905,1732539449246 for 'test_row_2' in 065eb16105c4aedcbfa7fb1ec2b14c86 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8249 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T13:00:09,776 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:09,776 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:09,776 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:09,901 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:09,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33764 deadline: 1732539669888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:09,902 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:09,902 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:09,902 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:09,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:09,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1732539669912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:09,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:09,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33782 deadline: 1732539669912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:09,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:09,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:09,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:09,920 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:09,920 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:09,920 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:09,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:09,960 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125aeb01705bb1c4151b7f0e0f3afc17c34_065eb16105c4aedcbfa7fb1ec2b14c86 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125aeb01705bb1c4151b7f0e0f3afc17c34_065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:09,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/4cea6c322769443b946345458855893a, store: [table=TestAcidGuarantees family=A region=065eb16105c4aedcbfa7fb1ec2b14c86] 2024-11-25T13:00:09,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/4cea6c322769443b946345458855893a is 175, key is test_row_0/A:col10/1732539609429/Put/seqid=0 2024-11-25T13:00:09,978 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 065eb16105c4aedcbfa7fb1ec2b14c86#A#compaction#494 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T13:00:09,979 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/2ddca797c1e94562a5f17432498e94c8 is 175, key is test_row_0/A:col10/1732539608425/Put/seqid=0 2024-11-25T13:00:09,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742415_1591 (size=48389) 2024-11-25T13:00:09,988 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=173, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/4cea6c322769443b946345458855893a 2024-11-25T13:00:09,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742416_1592 (size=31447) 2024-11-25T13:00:09,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/8a80df7c7c384bfe9d30d4c7783b0337 is 50, key is test_row_0/B:col10/1732539609429/Put/seqid=0 2024-11-25T13:00:10,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742417_1593 (size=12151) 2024-11-25T13:00:10,030 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/8a80df7c7c384bfe9d30d4c7783b0337 2024-11-25T13:00:10,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/ca4eb6741a4148fa84cb6ef250754549 is 50, key is test_row_0/C:col10/1732539609429/Put/seqid=0 2024-11-25T13:00:10,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742418_1594 (size=12151) 2024-11-25T13:00:10,094 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/ca4eb6741a4148fa84cb6ef250754549 2024-11-25T13:00:10,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/4cea6c322769443b946345458855893a as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/4cea6c322769443b946345458855893a 2024-11-25T13:00:10,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-25T13:00:10,123 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/4cea6c322769443b946345458855893a, entries=250, sequenceid=173, filesize=47.3 K 2024-11-25T13:00:10,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/8a80df7c7c384bfe9d30d4c7783b0337 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/8a80df7c7c384bfe9d30d4c7783b0337 2024-11-25T13:00:10,139 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/8a80df7c7c384bfe9d30d4c7783b0337, entries=150, sequenceid=173, filesize=11.9 K 2024-11-25T13:00:10,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/ca4eb6741a4148fa84cb6ef250754549 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/ca4eb6741a4148fa84cb6ef250754549 2024-11-25T13:00:10,183 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/ca4eb6741a4148fa84cb6ef250754549, entries=150, sequenceid=173, filesize=11.9 K 2024-11-25T13:00:10,187 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 065eb16105c4aedcbfa7fb1ec2b14c86 in 752ms, sequenceid=173, compaction requested=false 2024-11-25T13:00:10,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2603): Flush status journal for 065eb16105c4aedcbfa7fb1ec2b14c86: 2024-11-25T13:00:10,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:10,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147 2024-11-25T13:00:10,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4169): Remote procedure done, pid=147 2024-11-25T13:00:10,194 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-25T13:00:10,194 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2250 sec 2024-11-25T13:00:10,197 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees in 1.2350 sec 2024-11-25T13:00:10,215 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T13:00:10,215 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 065eb16105c4aedcbfa7fb1ec2b14c86 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-25T13:00:10,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:10,241 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125ec694a9bbd2f403ba6a1d52d45bc9026_065eb16105c4aedcbfa7fb1ec2b14c86 is 50, key is test_row_0/A:col10/1732539610214/Put/seqid=0 2024-11-25T13:00:10,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:10,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1732539670252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:10,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:10,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:10,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:10,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:10,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33782 deadline: 1732539670256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:10,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:10,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33764 deadline: 1732539670258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:10,262 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:10,262 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:10,262 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:10,262 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:10,262 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:10,262 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:10,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742419_1595 (size=14794) 2024-11-25T13:00:10,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:10,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33764 deadline: 1732539670370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:10,373 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:10,373 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:10,373 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:10,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:10,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33782 deadline: 1732539670370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:10,373 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:10,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:10,374 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:10,374 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:10,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1732539670370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:10,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:10,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:10,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:10,394 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/2ddca797c1e94562a5f17432498e94c8 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/2ddca797c1e94562a5f17432498e94c8 2024-11-25T13:00:10,399 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 065eb16105c4aedcbfa7fb1ec2b14c86/A of 065eb16105c4aedcbfa7fb1ec2b14c86 into 2ddca797c1e94562a5f17432498e94c8(size=30.7 K), total size for store is 78.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T13:00:10,399 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 065eb16105c4aedcbfa7fb1ec2b14c86: 2024-11-25T13:00:10,399 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., storeName=065eb16105c4aedcbfa7fb1ec2b14c86/A, priority=13, startTime=1732539609415; duration=0sec 2024-11-25T13:00:10,399 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T13:00:10,399 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 065eb16105c4aedcbfa7fb1ec2b14c86:A 2024-11-25T13:00:10,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:10,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1732539670580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:10,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:10,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:10,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:10,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:10,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33764 deadline: 1732539670580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:10,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:10,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:10,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:10,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:10,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33782 deadline: 1732539670580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:10,585 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:10,585 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:10,585 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:10,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:10,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33740 deadline: 1732539670600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:10,604 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 1ff8009e463e,35905,1732539449246 for 'test_row_1' in 065eb16105c4aedcbfa7fb1ec2b14c86 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T13:00:10,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:10,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:10,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:10,692 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:10,696 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125ec694a9bbd2f403ba6a1d52d45bc9026_065eb16105c4aedcbfa7fb1ec2b14c86 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125ec694a9bbd2f403ba6a1d52d45bc9026_065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:10,697 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/f5af782a93704dc49ccd218e18bbb3b2, store: [table=TestAcidGuarantees family=A region=065eb16105c4aedcbfa7fb1ec2b14c86] 2024-11-25T13:00:10,697 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/f5af782a93704dc49ccd218e18bbb3b2 is 175, key is test_row_0/A:col10/1732539610214/Put/seqid=0 2024-11-25T13:00:10,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742420_1596 (size=39749) 2024-11-25T13:00:10,745 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=199, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/f5af782a93704dc49ccd218e18bbb3b2 2024-11-25T13:00:10,764 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/c57a6e0c74b440f39069e5c058e97fe2 is 50, key is test_row_0/B:col10/1732539610214/Put/seqid=0 2024-11-25T13:00:10,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742421_1597 (size=12151) 2024-11-25T13:00:10,795 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/c57a6e0c74b440f39069e5c058e97fe2 2024-11-25T13:00:10,812 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/6b5a43ae991c48a3a45d4a6d78d80ecf is 50, key is test_row_0/C:col10/1732539610214/Put/seqid=0 2024-11-25T13:00:10,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742422_1598 (size=12151) 2024-11-25T13:00:10,861 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/6b5a43ae991c48a3a45d4a6d78d80ecf 2024-11-25T13:00:10,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/f5af782a93704dc49ccd218e18bbb3b2 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/f5af782a93704dc49ccd218e18bbb3b2 2024-11-25T13:00:10,878 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/f5af782a93704dc49ccd218e18bbb3b2, entries=200, sequenceid=199, filesize=38.8 K 2024-11-25T13:00:10,879 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/c57a6e0c74b440f39069e5c058e97fe2 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/c57a6e0c74b440f39069e5c058e97fe2 2024-11-25T13:00:10,884 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/c57a6e0c74b440f39069e5c058e97fe2, entries=150, sequenceid=199, filesize=11.9 K 2024-11-25T13:00:10,885 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/6b5a43ae991c48a3a45d4a6d78d80ecf as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/6b5a43ae991c48a3a45d4a6d78d80ecf 2024-11-25T13:00:10,893 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/6b5a43ae991c48a3a45d4a6d78d80ecf, entries=150, sequenceid=199, filesize=11.9 K 2024-11-25T13:00:10,893 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 065eb16105c4aedcbfa7fb1ec2b14c86 in 678ms, sequenceid=199, compaction requested=true 2024-11-25T13:00:10,893 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 065eb16105c4aedcbfa7fb1ec2b14c86: 2024-11-25T13:00:10,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 065eb16105c4aedcbfa7fb1ec2b14c86:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T13:00:10,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T13:00:10,894 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T13:00:10,894 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T13:00:10,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 065eb16105c4aedcbfa7fb1ec2b14c86:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T13:00:10,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T13:00:10,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 065eb16105c4aedcbfa7fb1ec2b14c86:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T13:00:10,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T13:00:10,895 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 119585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T13:00:10,895 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1541): 065eb16105c4aedcbfa7fb1ec2b14c86/A is initiating minor compaction (all files) 2024-11-25T13:00:10,895 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 065eb16105c4aedcbfa7fb1ec2b14c86/A in TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:10,895 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/2ddca797c1e94562a5f17432498e94c8, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/4cea6c322769443b946345458855893a, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/f5af782a93704dc49ccd218e18bbb3b2] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp, totalSize=116.8 K 2024-11-25T13:00:10,895 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:10,895 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. files: [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/2ddca797c1e94562a5f17432498e94c8, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/4cea6c322769443b946345458855893a, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/f5af782a93704dc49ccd218e18bbb3b2] 2024-11-25T13:00:10,895 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T13:00:10,895 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): 065eb16105c4aedcbfa7fb1ec2b14c86/B is initiating minor compaction (all files) 2024-11-25T13:00:10,895 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 065eb16105c4aedcbfa7fb1ec2b14c86/B in TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:10,895 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/65d601588e17457cb335f172d7d1832f, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/8a80df7c7c384bfe9d30d4c7783b0337, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/c57a6e0c74b440f39069e5c058e97fe2] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp, totalSize=35.9 K 2024-11-25T13:00:10,896 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ddca797c1e94562a5f17432498e94c8, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1732539608424 2024-11-25T13:00:10,896 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 65d601588e17457cb335f172d7d1832f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1732539608424 2024-11-25T13:00:10,896 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4cea6c322769443b946345458855893a, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732539608769 2024-11-25T13:00:10,896 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a80df7c7c384bfe9d30d4c7783b0337, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732539608776 2024-11-25T13:00:10,896 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5af782a93704dc49ccd218e18bbb3b2, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732539609506 2024-11-25T13:00:10,897 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting c57a6e0c74b440f39069e5c058e97fe2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732539609506 2024-11-25T13:00:10,897 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T13:00:10,897 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 065eb16105c4aedcbfa7fb1ec2b14c86 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-25T13:00:10,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:10,908 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=065eb16105c4aedcbfa7fb1ec2b14c86] 2024-11-25T13:00:10,909 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 065eb16105c4aedcbfa7fb1ec2b14c86#B#compaction#503 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T13:00:10,910 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/80f834c5e1134ceeb1fb5b4a9692e5a0 is 50, key is test_row_0/B:col10/1732539610214/Put/seqid=0 2024-11-25T13:00:10,916 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241125eeae82453b7c432a85a0b4407e4d45df_065eb16105c4aedcbfa7fb1ec2b14c86 store=[table=TestAcidGuarantees family=A region=065eb16105c4aedcbfa7fb1ec2b14c86] 2024-11-25T13:00:10,918 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241125eeae82453b7c432a85a0b4407e4d45df_065eb16105c4aedcbfa7fb1ec2b14c86, store=[table=TestAcidGuarantees family=A region=065eb16105c4aedcbfa7fb1ec2b14c86] 2024-11-25T13:00:10,918 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125eeae82453b7c432a85a0b4407e4d45df_065eb16105c4aedcbfa7fb1ec2b14c86 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=065eb16105c4aedcbfa7fb1ec2b14c86] 2024-11-25T13:00:10,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742424_1600 (size=4469) 2024-11-25T13:00:10,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742423_1599 (size=12595) 2024-11-25T13:00:10,974 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125a3940c8519b34aca99a3b380c7533869_065eb16105c4aedcbfa7fb1ec2b14c86 is 50, key is test_row_0/A:col10/1732539610256/Put/seqid=0 2024-11-25T13:00:10,983 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 065eb16105c4aedcbfa7fb1ec2b14c86#A#compaction#504 average throughput is 0.33 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T13:00:10,984 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/7049665fd88b448c9f277d56be1af121 is 175, key is test_row_0/A:col10/1732539610214/Put/seqid=0 2024-11-25T13:00:10,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:10,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33764 deadline: 1732539670977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:10,989 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:10,989 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:10,989 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:10,993 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/80f834c5e1134ceeb1fb5b4a9692e5a0 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/80f834c5e1134ceeb1fb5b4a9692e5a0 2024-11-25T13:00:10,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:10,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1732539670988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:10,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:10,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33782 deadline: 1732539670988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:10,997 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:10,997 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:10,997 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:10,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:10,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:10,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:11,019 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 065eb16105c4aedcbfa7fb1ec2b14c86/B of 065eb16105c4aedcbfa7fb1ec2b14c86 into 80f834c5e1134ceeb1fb5b4a9692e5a0(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T13:00:11,019 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 065eb16105c4aedcbfa7fb1ec2b14c86: 2024-11-25T13:00:11,020 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., storeName=065eb16105c4aedcbfa7fb1ec2b14c86/B, priority=13, startTime=1732539610894; duration=0sec 2024-11-25T13:00:11,020 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T13:00:11,020 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 065eb16105c4aedcbfa7fb1ec2b14c86:B 2024-11-25T13:00:11,020 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T13:00:11,022 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T13:00:11,022 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): 065eb16105c4aedcbfa7fb1ec2b14c86/C is initiating minor compaction (all files) 2024-11-25T13:00:11,022 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 065eb16105c4aedcbfa7fb1ec2b14c86/C in TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:11,023 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/ac11773566104d58ac71b3384593cc57, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/ca4eb6741a4148fa84cb6ef250754549, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/6b5a43ae991c48a3a45d4a6d78d80ecf] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp, totalSize=35.9 K 2024-11-25T13:00:11,023 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting ac11773566104d58ac71b3384593cc57, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1732539608424 2024-11-25T13:00:11,023 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting ca4eb6741a4148fa84cb6ef250754549, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732539608776 2024-11-25T13:00:11,024 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b5a43ae991c48a3a45d4a6d78d80ecf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732539609506 2024-11-25T13:00:11,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742425_1601 (size=14794) 2024-11-25T13:00:11,062 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 065eb16105c4aedcbfa7fb1ec2b14c86#C#compaction#506 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T13:00:11,063 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/f912f897630a4d068d4b9a9bc5eea475 is 50, key is test_row_0/C:col10/1732539610214/Put/seqid=0 2024-11-25T13:00:11,063 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:11,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742426_1602 (size=31549) 2024-11-25T13:00:11,072 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/7049665fd88b448c9f277d56be1af121 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/7049665fd88b448c9f277d56be1af121 2024-11-25T13:00:11,073 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125a3940c8519b34aca99a3b380c7533869_065eb16105c4aedcbfa7fb1ec2b14c86 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125a3940c8519b34aca99a3b380c7533869_065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:11,075 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/d79633c7018742b5afa9c5f8d1352e13, store: [table=TestAcidGuarantees family=A region=065eb16105c4aedcbfa7fb1ec2b14c86] 2024-11-25T13:00:11,075 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/d79633c7018742b5afa9c5f8d1352e13 is 175, key is test_row_0/A:col10/1732539610256/Put/seqid=0 2024-11-25T13:00:11,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742427_1603 (size=12595) 2024-11-25T13:00:11,080 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 065eb16105c4aedcbfa7fb1ec2b14c86/A of 065eb16105c4aedcbfa7fb1ec2b14c86 into 7049665fd88b448c9f277d56be1af121(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T13:00:11,080 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 065eb16105c4aedcbfa7fb1ec2b14c86: 2024-11-25T13:00:11,080 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., storeName=065eb16105c4aedcbfa7fb1ec2b14c86/A, priority=13, startTime=1732539610894; duration=0sec 2024-11-25T13:00:11,080 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T13:00:11,080 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 065eb16105c4aedcbfa7fb1ec2b14c86:A 2024-11-25T13:00:11,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742428_1604 (size=39749) 2024-11-25T13:00:11,082 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=213, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/d79633c7018742b5afa9c5f8d1352e13 2024-11-25T13:00:11,086 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/f912f897630a4d068d4b9a9bc5eea475 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/f912f897630a4d068d4b9a9bc5eea475 2024-11-25T13:00:11,090 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/689521bfe2ad4d15b86b33f2a53b618d is 50, key is test_row_0/B:col10/1732539610256/Put/seqid=0 2024-11-25T13:00:11,099 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 065eb16105c4aedcbfa7fb1ec2b14c86/C of 065eb16105c4aedcbfa7fb1ec2b14c86 into f912f897630a4d068d4b9a9bc5eea475(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T13:00:11,099 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 065eb16105c4aedcbfa7fb1ec2b14c86: 2024-11-25T13:00:11,099 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., storeName=065eb16105c4aedcbfa7fb1ec2b14c86/C, priority=13, startTime=1732539610894; duration=0sec 2024-11-25T13:00:11,099 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T13:00:11,100 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 065eb16105c4aedcbfa7fb1ec2b14c86:C 2024-11-25T13:00:11,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:11,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33764 deadline: 1732539671100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:11,104 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:11,104 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:11,104 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:11,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:11,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33782 deadline: 1732539671101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:11,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:11,105 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:11,105 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:11,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1732539671101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:11,105 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:11,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:11,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:11,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:11,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-25T13:00:11,115 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T13:00:11,115 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T13:00:11,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-11-25T13:00:11,117 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T13:00:11,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-25T13:00:11,118 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T13:00:11,118 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T13:00:11,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742429_1605 (size=12151) 2024-11-25T13:00:11,124 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/689521bfe2ad4d15b86b33f2a53b618d 2024-11-25T13:00:11,156 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/b12e02a3f6564b1a9b47ca43ae2626b2 is 50, key is test_row_0/C:col10/1732539610256/Put/seqid=0 2024-11-25T13:00:11,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742430_1606 (size=12151) 2024-11-25T13:00:11,186 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/b12e02a3f6564b1a9b47ca43ae2626b2 2024-11-25T13:00:11,191 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/d79633c7018742b5afa9c5f8d1352e13 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/d79633c7018742b5afa9c5f8d1352e13 2024-11-25T13:00:11,194 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/d79633c7018742b5afa9c5f8d1352e13, entries=200, sequenceid=213, filesize=38.8 K 2024-11-25T13:00:11,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/689521bfe2ad4d15b86b33f2a53b618d as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/689521bfe2ad4d15b86b33f2a53b618d 2024-11-25T13:00:11,198 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/689521bfe2ad4d15b86b33f2a53b618d, entries=150, sequenceid=213, filesize=11.9 K 2024-11-25T13:00:11,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/b12e02a3f6564b1a9b47ca43ae2626b2 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/b12e02a3f6564b1a9b47ca43ae2626b2 2024-11-25T13:00:11,203 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/b12e02a3f6564b1a9b47ca43ae2626b2, entries=150, sequenceid=213, filesize=11.9 K 2024-11-25T13:00:11,203 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 065eb16105c4aedcbfa7fb1ec2b14c86 in 306ms, sequenceid=213, compaction requested=false 2024-11-25T13:00:11,203 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 065eb16105c4aedcbfa7fb1ec2b14c86: 2024-11-25T13:00:11,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-25T13:00:11,278 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-25T13:00:11,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:11,278 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing 065eb16105c4aedcbfa7fb1ec2b14c86 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-25T13:00:11,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112562ed6d11882543f8b4200341d5e18879_065eb16105c4aedcbfa7fb1ec2b14c86 is 50, key is test_row_0/A:col10/1732539610977/Put/seqid=0 2024-11-25T13:00:11,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. as already flushing 2024-11-25T13:00:11,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:11,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742431_1607 (size=12304) 2024-11-25T13:00:11,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:11,360 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112562ed6d11882543f8b4200341d5e18879_065eb16105c4aedcbfa7fb1ec2b14c86 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112562ed6d11882543f8b4200341d5e18879_065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:11,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/ef2e4fc7e2aa44e2bf928f33bcefb473, store: [table=TestAcidGuarantees family=A region=065eb16105c4aedcbfa7fb1ec2b14c86] 2024-11-25T13:00:11,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/ef2e4fc7e2aa44e2bf928f33bcefb473 is 175, key is test_row_0/A:col10/1732539610977/Put/seqid=0 2024-11-25T13:00:11,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:11,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1732539671360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:11,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:11,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:11,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:11,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:11,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33764 deadline: 1732539671360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:11,372 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:11,372 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:11,372 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:11,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:11,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33782 deadline: 1732539671372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:11,375 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:11,375 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:11,375 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:11,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742432_1608 (size=31105) 2024-11-25T13:00:11,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-25T13:00:11,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:11,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33764 deadline: 1732539671483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:11,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:11,490 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:11,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1732539671483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:11,490 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:11,490 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:11,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:11,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:11,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:11,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:11,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33782 deadline: 1732539671484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:11,490 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:11,491 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:11,491 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:11,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:11,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1732539671700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:11,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:11,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:11,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33764 deadline: 1732539671700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:11,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:11,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:11,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:11,704 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:11,704 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:11,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33782 deadline: 1732539671700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:11,704 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:11,704 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:11,704 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:11,704 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:11,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-25T13:00:11,800 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=239, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/ef2e4fc7e2aa44e2bf928f33bcefb473 2024-11-25T13:00:11,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/2266e99c8d7147e8a93ed9741665d03b is 50, key is test_row_0/B:col10/1732539610977/Put/seqid=0 2024-11-25T13:00:11,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742433_1609 (size=12151) 2024-11-25T13:00:11,828 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/2266e99c8d7147e8a93ed9741665d03b 2024-11-25T13:00:11,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/394b57743e5347f69e9a9824f1da4c26 is 50, key is test_row_0/C:col10/1732539610977/Put/seqid=0 2024-11-25T13:00:11,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742434_1610 (size=12151) 2024-11-25T13:00:11,866 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/394b57743e5347f69e9a9824f1da4c26 2024-11-25T13:00:11,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/ef2e4fc7e2aa44e2bf928f33bcefb473 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/ef2e4fc7e2aa44e2bf928f33bcefb473 2024-11-25T13:00:11,883 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/ef2e4fc7e2aa44e2bf928f33bcefb473, entries=150, sequenceid=239, filesize=30.4 K 2024-11-25T13:00:11,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/2266e99c8d7147e8a93ed9741665d03b as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/2266e99c8d7147e8a93ed9741665d03b 2024-11-25T13:00:11,889 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/2266e99c8d7147e8a93ed9741665d03b, entries=150, sequenceid=239, filesize=11.9 K 2024-11-25T13:00:11,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/394b57743e5347f69e9a9824f1da4c26 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/394b57743e5347f69e9a9824f1da4c26 2024-11-25T13:00:11,895 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/394b57743e5347f69e9a9824f1da4c26, entries=150, sequenceid=239, filesize=11.9 K 2024-11-25T13:00:11,896 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for 065eb16105c4aedcbfa7fb1ec2b14c86 in 618ms, sequenceid=239, compaction requested=true 2024-11-25T13:00:11,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for 065eb16105c4aedcbfa7fb1ec2b14c86: 2024-11-25T13:00:11,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:11,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-11-25T13:00:11,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-11-25T13:00:11,903 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-25T13:00:11,903 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 780 msec 2024-11-25T13:00:11,906 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 788 msec 2024-11-25T13:00:12,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:12,014 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T13:00:12,014 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 065eb16105c4aedcbfa7fb1ec2b14c86 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-25T13:00:12,041 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411251c935c2fea814d1db1a0a867d1a7b1e5_065eb16105c4aedcbfa7fb1ec2b14c86 is 50, key is test_row_0/A:col10/1732539612013/Put/seqid=0 2024-11-25T13:00:12,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:12,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33764 deadline: 1732539672056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:12,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:12,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33782 deadline: 1732539672057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:12,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:12,065 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:12,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1732539672058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:12,065 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:12,065 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:12,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:12,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:12,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:12,066 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:12,066 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:12,066 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:12,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742435_1611 (size=17284) 2024-11-25T13:00:12,076 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:12,079 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411251c935c2fea814d1db1a0a867d1a7b1e5_065eb16105c4aedcbfa7fb1ec2b14c86 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411251c935c2fea814d1db1a0a867d1a7b1e5_065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:12,080 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/edf64eadc2f64469b146d56748034ae9, store: [table=TestAcidGuarantees family=A region=065eb16105c4aedcbfa7fb1ec2b14c86] 2024-11-25T13:00:12,081 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/edf64eadc2f64469b146d56748034ae9 is 175, key is test_row_0/A:col10/1732539612013/Put/seqid=0 2024-11-25T13:00:12,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742436_1612 (size=48389) 2024-11-25T13:00:12,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:12,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33782 deadline: 1732539672170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:12,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:12,174 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:12,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1732539672170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:12,174 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:12,175 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:12,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:12,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:12,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:12,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33764 deadline: 1732539672170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:12,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:12,175 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:12,175 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:12,175 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:12,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-25T13:00:12,250 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T13:00:12,251 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T13:00:12,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-25T13:00:12,254 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T13:00:12,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-25T13:00:12,260 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T13:00:12,261 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T13:00:12,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-25T13:00:12,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:12,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1732539672384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:12,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:12,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33764 deadline: 1732539672384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:12,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:12,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33782 deadline: 1732539672384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:12,389 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:12,389 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:12,389 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:12,391 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:12,391 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:12,391 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:12,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:12,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:12,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:12,416 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-25T13:00:12,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:12,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. as already flushing 2024-11-25T13:00:12,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:12,416 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:12,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:12,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:12,522 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=255, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/edf64eadc2f64469b146d56748034ae9 2024-11-25T13:00:12,529 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/157862308be74a0aba128e555e4bba2f is 50, key is test_row_0/B:col10/1732539612013/Put/seqid=0 2024-11-25T13:00:12,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742437_1613 (size=12151) 2024-11-25T13:00:12,536 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/157862308be74a0aba128e555e4bba2f 2024-11-25T13:00:12,542 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/102712bae900483284798a98e5133a37 is 50, key is test_row_0/C:col10/1732539612013/Put/seqid=0 2024-11-25T13:00:12,580 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-25T13:00:12,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:12,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. as already flushing 2024-11-25T13:00:12,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:12,580 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:12,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:12,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-25T13:00:12,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:12,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742438_1614 (size=12151) 2024-11-25T13:00:12,598 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/102712bae900483284798a98e5133a37 2024-11-25T13:00:12,606 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/edf64eadc2f64469b146d56748034ae9 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/edf64eadc2f64469b146d56748034ae9 2024-11-25T13:00:12,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:12,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33740 deadline: 1732539672615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:12,618 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 1ff8009e463e,35905,1732539449246 for 'test_row_1' in 065eb16105c4aedcbfa7fb1ec2b14c86 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T13:00:12,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:12,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:12,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:12,631 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/edf64eadc2f64469b146d56748034ae9, entries=250, sequenceid=255, filesize=47.3 K 2024-11-25T13:00:12,632 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/157862308be74a0aba128e555e4bba2f as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/157862308be74a0aba128e555e4bba2f 2024-11-25T13:00:12,636 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/157862308be74a0aba128e555e4bba2f, entries=150, sequenceid=255, filesize=11.9 K 2024-11-25T13:00:12,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/102712bae900483284798a98e5133a37 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/102712bae900483284798a98e5133a37 2024-11-25T13:00:12,642 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/102712bae900483284798a98e5133a37, entries=150, sequenceid=255, filesize=11.9 K 2024-11-25T13:00:12,643 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for 065eb16105c4aedcbfa7fb1ec2b14c86 in 629ms, sequenceid=255, compaction requested=true 2024-11-25T13:00:12,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 065eb16105c4aedcbfa7fb1ec2b14c86: 2024-11-25T13:00:12,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 065eb16105c4aedcbfa7fb1ec2b14c86:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T13:00:12,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T13:00:12,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 065eb16105c4aedcbfa7fb1ec2b14c86:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T13:00:12,644 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T13:00:12,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T13:00:12,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 065eb16105c4aedcbfa7fb1ec2b14c86:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T13:00:12,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-25T13:00:12,644 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T13:00:12,645 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T13:00:12,645 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): 065eb16105c4aedcbfa7fb1ec2b14c86/B is initiating minor compaction (all files) 2024-11-25T13:00:12,645 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 065eb16105c4aedcbfa7fb1ec2b14c86/B in TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:12,646 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/80f834c5e1134ceeb1fb5b4a9692e5a0, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/689521bfe2ad4d15b86b33f2a53b618d, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/2266e99c8d7147e8a93ed9741665d03b, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/157862308be74a0aba128e555e4bba2f] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp, totalSize=47.9 K 2024-11-25T13:00:12,646 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 150792 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T13:00:12,646 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1541): 065eb16105c4aedcbfa7fb1ec2b14c86/A is initiating minor compaction (all files) 2024-11-25T13:00:12,646 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 065eb16105c4aedcbfa7fb1ec2b14c86/A in TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:12,646 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/7049665fd88b448c9f277d56be1af121, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/d79633c7018742b5afa9c5f8d1352e13, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/ef2e4fc7e2aa44e2bf928f33bcefb473, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/edf64eadc2f64469b146d56748034ae9] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp, totalSize=147.3 K 2024-11-25T13:00:12,646 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:12,646 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. files: [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/7049665fd88b448c9f277d56be1af121, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/d79633c7018742b5afa9c5f8d1352e13, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/ef2e4fc7e2aa44e2bf928f33bcefb473, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/edf64eadc2f64469b146d56748034ae9] 2024-11-25T13:00:12,646 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7049665fd88b448c9f277d56be1af121, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732539609506 2024-11-25T13:00:12,646 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 80f834c5e1134ceeb1fb5b4a9692e5a0, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732539609506 2024-11-25T13:00:12,646 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 689521bfe2ad4d15b86b33f2a53b618d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1732539610254 2024-11-25T13:00:12,646 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting d79633c7018742b5afa9c5f8d1352e13, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1732539610247 2024-11-25T13:00:12,647 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 2266e99c8d7147e8a93ed9741665d03b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1732539610936 2024-11-25T13:00:12,647 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef2e4fc7e2aa44e2bf928f33bcefb473, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1732539610936 2024-11-25T13:00:12,647 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 157862308be74a0aba128e555e4bba2f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732539611334 2024-11-25T13:00:12,647 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting edf64eadc2f64469b146d56748034ae9, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732539611332 2024-11-25T13:00:12,659 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=065eb16105c4aedcbfa7fb1ec2b14c86] 2024-11-25T13:00:12,670 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 065eb16105c4aedcbfa7fb1ec2b14c86#B#compaction#516 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T13:00:12,670 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/a83fe6beadb6421f988d45c664b90c22 is 50, key is test_row_0/B:col10/1732539612013/Put/seqid=0 2024-11-25T13:00:12,692 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241125e94578fc7d28416ab36962960be68e21_065eb16105c4aedcbfa7fb1ec2b14c86 store=[table=TestAcidGuarantees family=A region=065eb16105c4aedcbfa7fb1ec2b14c86] 2024-11-25T13:00:12,694 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241125e94578fc7d28416ab36962960be68e21_065eb16105c4aedcbfa7fb1ec2b14c86, store=[table=TestAcidGuarantees family=A region=065eb16105c4aedcbfa7fb1ec2b14c86] 2024-11-25T13:00:12,694 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125e94578fc7d28416ab36962960be68e21_065eb16105c4aedcbfa7fb1ec2b14c86 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=065eb16105c4aedcbfa7fb1ec2b14c86] 2024-11-25T13:00:12,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:12,704 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T13:00:12,704 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 065eb16105c4aedcbfa7fb1ec2b14c86 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-25T13:00:12,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742439_1615 (size=12731) 2024-11-25T13:00:12,731 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411250b186797ba3f4fcaacd663c47d10035c_065eb16105c4aedcbfa7fb1ec2b14c86 is 50, key is test_row_0/A:col10/1732539612703/Put/seqid=0 2024-11-25T13:00:12,737 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-25T13:00:12,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:12,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. as already flushing 2024-11-25T13:00:12,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:12,737 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:12,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:12,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:12,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:12,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33764 deadline: 1732539672732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:12,743 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:12,743 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:12,743 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:12,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:12,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1732539672736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:12,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:12,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:12,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:12,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:12,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33782 deadline: 1732539672738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:12,744 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:12,744 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:12,744 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:12,745 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/a83fe6beadb6421f988d45c664b90c22 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/a83fe6beadb6421f988d45c664b90c22 2024-11-25T13:00:12,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742440_1616 (size=4469) 2024-11-25T13:00:12,751 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 065eb16105c4aedcbfa7fb1ec2b14c86#A#compaction#515 average throughput is 0.27 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T13:00:12,752 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/f5e3d78c0ca0497bb52bafe1322a4985 is 175, key is test_row_0/A:col10/1732539612013/Put/seqid=0 2024-11-25T13:00:12,767 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 065eb16105c4aedcbfa7fb1ec2b14c86/B of 065eb16105c4aedcbfa7fb1ec2b14c86 into a83fe6beadb6421f988d45c664b90c22(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T13:00:12,767 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 065eb16105c4aedcbfa7fb1ec2b14c86: 2024-11-25T13:00:12,767 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., storeName=065eb16105c4aedcbfa7fb1ec2b14c86/B, priority=12, startTime=1732539612644; duration=0sec 2024-11-25T13:00:12,767 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T13:00:12,767 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 065eb16105c4aedcbfa7fb1ec2b14c86:B 2024-11-25T13:00:12,767 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T13:00:12,768 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T13:00:12,768 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): 065eb16105c4aedcbfa7fb1ec2b14c86/C is initiating minor compaction (all files) 2024-11-25T13:00:12,768 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 065eb16105c4aedcbfa7fb1ec2b14c86/C in TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:12,768 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/f912f897630a4d068d4b9a9bc5eea475, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/b12e02a3f6564b1a9b47ca43ae2626b2, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/394b57743e5347f69e9a9824f1da4c26, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/102712bae900483284798a98e5133a37] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp, totalSize=47.9 K 2024-11-25T13:00:12,769 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting f912f897630a4d068d4b9a9bc5eea475, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732539609506 2024-11-25T13:00:12,769 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting b12e02a3f6564b1a9b47ca43ae2626b2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1732539610254 2024-11-25T13:00:12,769 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 394b57743e5347f69e9a9824f1da4c26, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1732539610936 2024-11-25T13:00:12,769 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 102712bae900483284798a98e5133a37, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732539611334 2024-11-25T13:00:12,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742441_1617 (size=17534) 2024-11-25T13:00:12,778 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 065eb16105c4aedcbfa7fb1ec2b14c86#C#compaction#518 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T13:00:12,779 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/646aae351bbd4c54b84aa6a6e04db69e is 50, key is test_row_0/C:col10/1732539612013/Put/seqid=0 2024-11-25T13:00:12,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742442_1618 (size=31685) 2024-11-25T13:00:12,780 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:12,784 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411250b186797ba3f4fcaacd663c47d10035c_065eb16105c4aedcbfa7fb1ec2b14c86 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411250b186797ba3f4fcaacd663c47d10035c_065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:12,785 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/ec95b895116a443d881d96facd416497, store: [table=TestAcidGuarantees family=A region=065eb16105c4aedcbfa7fb1ec2b14c86] 2024-11-25T13:00:12,786 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/ec95b895116a443d881d96facd416497 is 175, key is test_row_0/A:col10/1732539612703/Put/seqid=0 2024-11-25T13:00:12,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742444_1620 (size=48639) 2024-11-25T13:00:12,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742443_1619 (size=12731) 2024-11-25T13:00:12,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:12,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1732539672850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:12,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:12,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:12,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:12,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:12,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33782 deadline: 1732539672850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:12,854 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:12,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:12,854 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:12,854 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:12,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33764 deadline: 1732539672850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:12,854 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:12,854 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:12,854 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:12,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-25T13:00:12,892 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-25T13:00:12,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:12,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. as already flushing 2024-11-25T13:00:12,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:12,892 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:12,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:12,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:13,057 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-25T13:00:13,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:13,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. as already flushing 2024-11-25T13:00:13,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:13,058 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:13,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:13,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:13,068 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:13,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1732539673061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:13,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:13,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:13,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:13,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:13,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:13,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33782 deadline: 1732539673061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:13,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33764 deadline: 1732539673061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:13,069 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:13,069 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:13,069 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:13,071 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:13,071 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:13,071 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:13,184 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/f5e3d78c0ca0497bb52bafe1322a4985 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/f5e3d78c0ca0497bb52bafe1322a4985 2024-11-25T13:00:13,188 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 065eb16105c4aedcbfa7fb1ec2b14c86/A of 065eb16105c4aedcbfa7fb1ec2b14c86 into f5e3d78c0ca0497bb52bafe1322a4985(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T13:00:13,188 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 065eb16105c4aedcbfa7fb1ec2b14c86: 2024-11-25T13:00:13,188 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., storeName=065eb16105c4aedcbfa7fb1ec2b14c86/A, priority=12, startTime=1732539612644; duration=0sec 2024-11-25T13:00:13,188 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T13:00:13,188 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 065eb16105c4aedcbfa7fb1ec2b14c86:A 2024-11-25T13:00:13,213 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-25T13:00:13,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:13,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. as already flushing 2024-11-25T13:00:13,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:13,213 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:13,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:13,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:13,244 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=278, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/ec95b895116a443d881d96facd416497 2024-11-25T13:00:13,250 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/f63048269c274fbeb02a501b7d42e592 is 50, key is test_row_0/B:col10/1732539612703/Put/seqid=0 2024-11-25T13:00:13,256 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/646aae351bbd4c54b84aa6a6e04db69e as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/646aae351bbd4c54b84aa6a6e04db69e 2024-11-25T13:00:13,260 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 065eb16105c4aedcbfa7fb1ec2b14c86/C of 065eb16105c4aedcbfa7fb1ec2b14c86 into 646aae351bbd4c54b84aa6a6e04db69e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T13:00:13,260 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 065eb16105c4aedcbfa7fb1ec2b14c86: 2024-11-25T13:00:13,260 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., storeName=065eb16105c4aedcbfa7fb1ec2b14c86/C, priority=12, startTime=1732539612644; duration=0sec 2024-11-25T13:00:13,260 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T13:00:13,260 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 065eb16105c4aedcbfa7fb1ec2b14c86:C 2024-11-25T13:00:13,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742445_1621 (size=12301) 2024-11-25T13:00:13,277 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/f63048269c274fbeb02a501b7d42e592 2024-11-25T13:00:13,286 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/b66cc7274d494fd2b929d6752b58d249 is 50, key is test_row_0/C:col10/1732539612703/Put/seqid=0 2024-11-25T13:00:13,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742446_1622 (size=12301) 2024-11-25T13:00:13,317 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/b66cc7274d494fd2b929d6752b58d249 2024-11-25T13:00:13,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/ec95b895116a443d881d96facd416497 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/ec95b895116a443d881d96facd416497 2024-11-25T13:00:13,331 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/ec95b895116a443d881d96facd416497, entries=250, sequenceid=278, filesize=47.5 K 2024-11-25T13:00:13,332 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/f63048269c274fbeb02a501b7d42e592 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/f63048269c274fbeb02a501b7d42e592 2024-11-25T13:00:13,338 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/f63048269c274fbeb02a501b7d42e592, entries=150, sequenceid=278, filesize=12.0 K 2024-11-25T13:00:13,338 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/b66cc7274d494fd2b929d6752b58d249 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/b66cc7274d494fd2b929d6752b58d249 2024-11-25T13:00:13,345 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/b66cc7274d494fd2b929d6752b58d249, entries=150, sequenceid=278, filesize=12.0 K 2024-11-25T13:00:13,346 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 065eb16105c4aedcbfa7fb1ec2b14c86 in 642ms, sequenceid=278, compaction requested=false 2024-11-25T13:00:13,346 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 065eb16105c4aedcbfa7fb1ec2b14c86: 2024-11-25T13:00:13,366 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-25T13:00:13,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:13,367 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing 065eb16105c4aedcbfa7fb1ec2b14c86 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-25T13:00:13,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125980783caecf74820af244ee33cd10574_065eb16105c4aedcbfa7fb1ec2b14c86 is 50, key is test_row_0/A:col10/1732539612717/Put/seqid=0 2024-11-25T13:00:13,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:13,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. as already flushing 2024-11-25T13:00:13,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742447_1623 (size=12454) 2024-11-25T13:00:13,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:13,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-25T13:00:13,403 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125980783caecf74820af244ee33cd10574_065eb16105c4aedcbfa7fb1ec2b14c86 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125980783caecf74820af244ee33cd10574_065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:13,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/8b9b8b5a969e40e8a07bf94fcd1b4135, store: [table=TestAcidGuarantees family=A region=065eb16105c4aedcbfa7fb1ec2b14c86] 2024-11-25T13:00:13,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/8b9b8b5a969e40e8a07bf94fcd1b4135 is 175, key is test_row_0/A:col10/1732539612717/Put/seqid=0 2024-11-25T13:00:13,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742448_1624 (size=31255) 2024-11-25T13:00:13,420 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=294, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/8b9b8b5a969e40e8a07bf94fcd1b4135 2024-11-25T13:00:13,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/1b9c70ff633840049de5c332838876bf is 50, key is test_row_0/B:col10/1732539612717/Put/seqid=0 2024-11-25T13:00:13,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:13,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33782 deadline: 1732539673429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:13,433 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:13,433 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:13,433 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:13,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742449_1625 (size=12301) 2024-11-25T13:00:13,434 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/1b9c70ff633840049de5c332838876bf 2024-11-25T13:00:13,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:13,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1732539673430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:13,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:13,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33764 deadline: 1732539673431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:13,436 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:13,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:13,436 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:13,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:13,436 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:13,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:13,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/ba2f61c5cc914c73ada04671ae67aab3 is 50, key is test_row_0/C:col10/1732539612717/Put/seqid=0 2024-11-25T13:00:13,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742450_1626 (size=12301) 2024-11-25T13:00:13,448 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/ba2f61c5cc914c73ada04671ae67aab3 2024-11-25T13:00:13,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/8b9b8b5a969e40e8a07bf94fcd1b4135 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/8b9b8b5a969e40e8a07bf94fcd1b4135 2024-11-25T13:00:13,457 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/8b9b8b5a969e40e8a07bf94fcd1b4135, entries=150, sequenceid=294, filesize=30.5 K 2024-11-25T13:00:13,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/1b9c70ff633840049de5c332838876bf as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/1b9c70ff633840049de5c332838876bf 2024-11-25T13:00:13,462 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/1b9c70ff633840049de5c332838876bf, entries=150, sequenceid=294, filesize=12.0 K 2024-11-25T13:00:13,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/ba2f61c5cc914c73ada04671ae67aab3 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/ba2f61c5cc914c73ada04671ae67aab3 2024-11-25T13:00:13,475 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/ba2f61c5cc914c73ada04671ae67aab3, entries=150, sequenceid=294, filesize=12.0 K 2024-11-25T13:00:13,480 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 065eb16105c4aedcbfa7fb1ec2b14c86 in 113ms, sequenceid=294, compaction requested=true 2024-11-25T13:00:13,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for 065eb16105c4aedcbfa7fb1ec2b14c86: 2024-11-25T13:00:13,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:13,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-25T13:00:13,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-25T13:00:13,485 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-25T13:00:13,485 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2240 sec 2024-11-25T13:00:13,487 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 1.2350 sec 2024-11-25T13:00:13,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:13,542 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T13:00:13,542 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 065eb16105c4aedcbfa7fb1ec2b14c86 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-25T13:00:13,549 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411258250882379e14b0da4668b1ba45330bd_065eb16105c4aedcbfa7fb1ec2b14c86 is 50, key is test_row_0/A:col10/1732539613429/Put/seqid=0 2024-11-25T13:00:13,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:13,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1732539673560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:13,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:13,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33764 deadline: 1732539673562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:13,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:13,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:13,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33782 deadline: 1732539673564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:13,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:13,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:13,568 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:13,568 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:13,568 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:13,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:13,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:13,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:13,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742451_1627 (size=14994) 2024-11-25T13:00:13,578 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:13,594 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411258250882379e14b0da4668b1ba45330bd_065eb16105c4aedcbfa7fb1ec2b14c86 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411258250882379e14b0da4668b1ba45330bd_065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:13,600 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/fb97de48734c4b22b3c5ad4dd1c2e73b, store: [table=TestAcidGuarantees family=A region=065eb16105c4aedcbfa7fb1ec2b14c86] 2024-11-25T13:00:13,601 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/fb97de48734c4b22b3c5ad4dd1c2e73b is 175, key is test_row_0/A:col10/1732539613429/Put/seqid=0 2024-11-25T13:00:13,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742452_1628 (size=39949) 2024-11-25T13:00:13,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:13,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33782 deadline: 1732539673672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:13,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:13,676 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:13,676 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:13,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33764 deadline: 1732539673672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:13,676 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:13,676 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:13,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:13,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:13,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:13,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1732539673683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:13,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:13,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:13,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:13,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:13,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33782 deadline: 1732539673881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:13,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:13,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33764 deadline: 1732539673881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:13,885 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:13,885 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:13,885 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:13,885 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:13,885 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:13,885 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:13,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:13,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1732539673900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:13,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:13,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:13,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:14,051 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=318, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/fb97de48734c4b22b3c5ad4dd1c2e73b 2024-11-25T13:00:14,065 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/d181f40310ca463284c6d715fdb69413 is 50, key is test_row_0/B:col10/1732539613429/Put/seqid=0 2024-11-25T13:00:14,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742453_1629 (size=12301) 2024-11-25T13:00:14,127 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/d181f40310ca463284c6d715fdb69413 2024-11-25T13:00:14,153 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/33eeac8f284f49d88cda8d3bae9a7fd2 is 50, key is test_row_0/C:col10/1732539613429/Put/seqid=0 2024-11-25T13:00:14,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742454_1630 (size=12301) 2024-11-25T13:00:14,185 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/33eeac8f284f49d88cda8d3bae9a7fd2 2024-11-25T13:00:14,196 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/fb97de48734c4b22b3c5ad4dd1c2e73b as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/fb97de48734c4b22b3c5ad4dd1c2e73b 2024-11-25T13:00:14,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:14,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33782 deadline: 1732539674192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:14,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:14,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33764 deadline: 1732539674195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:14,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:14,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:14,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:14,201 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:14,201 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:14,201 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:14,205 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/fb97de48734c4b22b3c5ad4dd1c2e73b, entries=200, sequenceid=318, filesize=39.0 K 2024-11-25T13:00:14,206 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/d181f40310ca463284c6d715fdb69413 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/d181f40310ca463284c6d715fdb69413 2024-11-25T13:00:14,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:14,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1732539674212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:14,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:14,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:14,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:14,227 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/d181f40310ca463284c6d715fdb69413, entries=150, sequenceid=318, filesize=12.0 K 2024-11-25T13:00:14,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/33eeac8f284f49d88cda8d3bae9a7fd2 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/33eeac8f284f49d88cda8d3bae9a7fd2 2024-11-25T13:00:14,267 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/33eeac8f284f49d88cda8d3bae9a7fd2, entries=150, sequenceid=318, filesize=12.0 K 2024-11-25T13:00:14,268 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 065eb16105c4aedcbfa7fb1ec2b14c86 in 726ms, sequenceid=318, compaction requested=true 2024-11-25T13:00:14,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 065eb16105c4aedcbfa7fb1ec2b14c86: 2024-11-25T13:00:14,269 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T13:00:14,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 065eb16105c4aedcbfa7fb1ec2b14c86:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T13:00:14,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T13:00:14,269 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T13:00:14,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 065eb16105c4aedcbfa7fb1ec2b14c86:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T13:00:14,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T13:00:14,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 065eb16105c4aedcbfa7fb1ec2b14c86:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T13:00:14,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T13:00:14,271 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 151528 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T13:00:14,271 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1541): 065eb16105c4aedcbfa7fb1ec2b14c86/A is initiating minor compaction (all files) 2024-11-25T13:00:14,272 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 065eb16105c4aedcbfa7fb1ec2b14c86/A in TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:14,272 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/f5e3d78c0ca0497bb52bafe1322a4985, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/ec95b895116a443d881d96facd416497, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/8b9b8b5a969e40e8a07bf94fcd1b4135, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/fb97de48734c4b22b3c5ad4dd1c2e73b] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp, totalSize=148.0 K 2024-11-25T13:00:14,272 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:14,272 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. files: [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/f5e3d78c0ca0497bb52bafe1322a4985, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/ec95b895116a443d881d96facd416497, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/8b9b8b5a969e40e8a07bf94fcd1b4135, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/fb97de48734c4b22b3c5ad4dd1c2e73b] 2024-11-25T13:00:14,272 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49634 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T13:00:14,272 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): 065eb16105c4aedcbfa7fb1ec2b14c86/B is initiating minor compaction (all files) 2024-11-25T13:00:14,272 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 065eb16105c4aedcbfa7fb1ec2b14c86/B in TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:14,272 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/a83fe6beadb6421f988d45c664b90c22, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/f63048269c274fbeb02a501b7d42e592, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/1b9c70ff633840049de5c332838876bf, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/d181f40310ca463284c6d715fdb69413] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp, totalSize=48.5 K 2024-11-25T13:00:14,272 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting a83fe6beadb6421f988d45c664b90c22, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732539611334 2024-11-25T13:00:14,272 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5e3d78c0ca0497bb52bafe1322a4985, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732539611334 2024-11-25T13:00:14,273 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting f63048269c274fbeb02a501b7d42e592, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732539612054 2024-11-25T13:00:14,274 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec95b895116a443d881d96facd416497, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732539612054 2024-11-25T13:00:14,274 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b9c70ff633840049de5c332838876bf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1732539612717 2024-11-25T13:00:14,275 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b9b8b5a969e40e8a07bf94fcd1b4135, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1732539612717 2024-11-25T13:00:14,276 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting d181f40310ca463284c6d715fdb69413, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1732539613425 2024-11-25T13:00:14,276 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb97de48734c4b22b3c5ad4dd1c2e73b, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1732539613408 2024-11-25T13:00:14,326 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 065eb16105c4aedcbfa7fb1ec2b14c86#B#compaction#527 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T13:00:14,326 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/e797a4811a274412bf15f54ce9a3d1ee is 50, key is test_row_0/B:col10/1732539613429/Put/seqid=0 2024-11-25T13:00:14,332 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=065eb16105c4aedcbfa7fb1ec2b14c86] 2024-11-25T13:00:14,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742455_1631 (size=13017) 2024-11-25T13:00:14,380 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241125ddfbcafd39f5423cbfb32ff0413c1bcf_065eb16105c4aedcbfa7fb1ec2b14c86 store=[table=TestAcidGuarantees family=A region=065eb16105c4aedcbfa7fb1ec2b14c86] 2024-11-25T13:00:14,382 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241125ddfbcafd39f5423cbfb32ff0413c1bcf_065eb16105c4aedcbfa7fb1ec2b14c86, store=[table=TestAcidGuarantees family=A region=065eb16105c4aedcbfa7fb1ec2b14c86] 2024-11-25T13:00:14,383 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125ddfbcafd39f5423cbfb32ff0413c1bcf_065eb16105c4aedcbfa7fb1ec2b14c86 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=065eb16105c4aedcbfa7fb1ec2b14c86] 2024-11-25T13:00:14,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742456_1632 (size=4469) 2024-11-25T13:00:14,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-25T13:00:14,411 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T13:00:14,411 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T13:00:14,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-25T13:00:14,413 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T13:00:14,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-25T13:00:14,414 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T13:00:14,414 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T13:00:14,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-25T13:00:14,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-25T13:00:14,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:14,566 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 065eb16105c4aedcbfa7fb1ec2b14c86 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-25T13:00:14,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411253a0f50a6a0cf48b6adafaafd9d2c9579_065eb16105c4aedcbfa7fb1ec2b14c86 is 50, key is test_row_0/A:col10/1732539613558/Put/seqid=0 2024-11-25T13:00:14,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742457_1633 (size=12454) 2024-11-25T13:00:14,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,585 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411253a0f50a6a0cf48b6adafaafd9d2c9579_065eb16105c4aedcbfa7fb1ec2b14c86 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411253a0f50a6a0cf48b6adafaafd9d2c9579_065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:14,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/0af3647a9f194c8d8acabadf61612cd6, store: [table=TestAcidGuarantees family=A region=065eb16105c4aedcbfa7fb1ec2b14c86] 2024-11-25T13:00:14,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/0af3647a9f194c8d8acabadf61612cd6 is 175, key is test_row_0/A:col10/1732539613558/Put/seqid=0 2024-11-25T13:00:14,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742458_1634 (size=31255) 2024-11-25T13:00:14,598 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=330, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/0af3647a9f194c8d8acabadf61612cd6 2024-11-25T13:00:14,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/d8c71ab023c440c7bde170e89573dbbb is 50, key is test_row_0/B:col10/1732539613558/Put/seqid=0 2024-11-25T13:00:14,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742459_1635 (size=12301) 2024-11-25T13:00:14,613 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/d8c71ab023c440c7bde170e89573dbbb 2024-11-25T13:00:14,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/e43622462eab46a886ac24aea6041446 is 50, key is test_row_0/C:col10/1732539613558/Put/seqid=0 2024-11-25T13:00:14,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742460_1636 (size=12301) 2024-11-25T13:00:14,636 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/e43622462eab46a886ac24aea6041446 2024-11-25T13:00:14,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/0af3647a9f194c8d8acabadf61612cd6 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/0af3647a9f194c8d8acabadf61612cd6 2024-11-25T13:00:14,647 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/0af3647a9f194c8d8acabadf61612cd6, entries=150, sequenceid=330, filesize=30.5 K 2024-11-25T13:00:14,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/d8c71ab023c440c7bde170e89573dbbb as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/d8c71ab023c440c7bde170e89573dbbb 2024-11-25T13:00:14,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,652 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/d8c71ab023c440c7bde170e89573dbbb, entries=150, sequenceid=330, filesize=12.0 K 2024-11-25T13:00:14,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/e43622462eab46a886ac24aea6041446 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/e43622462eab46a886ac24aea6041446 2024-11-25T13:00:14,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,656 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/e43622462eab46a886ac24aea6041446, entries=150, sequenceid=330, filesize=12.0 K 2024-11-25T13:00:14,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,657 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=0 B/0 for 065eb16105c4aedcbfa7fb1ec2b14c86 in 91ms, sequenceid=330, compaction requested=true 2024-11-25T13:00:14,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 065eb16105c4aedcbfa7fb1ec2b14c86: 2024-11-25T13:00:14,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:14,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-25T13:00:14,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-25T13:00:14,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,660 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-25T13:00:14,660 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 244 msec 2024-11-25T13:00:14,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,662 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 250 msec 2024-11-25T13:00:14,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-25T13:00:14,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,730 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T13:00:14,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,730 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T13:00:14,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-25T13:00:14,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,732 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T13:00:14,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-25T13:00:14,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,733 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T13:00:14,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,733 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T13:00:14,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,778 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/e797a4811a274412bf15f54ce9a3d1ee as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/e797a4811a274412bf15f54ce9a3d1ee 2024-11-25T13:00:14,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,783 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 065eb16105c4aedcbfa7fb1ec2b14c86/B of 065eb16105c4aedcbfa7fb1ec2b14c86 into e797a4811a274412bf15f54ce9a3d1ee(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T13:00:14,783 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 065eb16105c4aedcbfa7fb1ec2b14c86: 2024-11-25T13:00:14,783 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., storeName=065eb16105c4aedcbfa7fb1ec2b14c86/B, priority=12, startTime=1732539614269; duration=0sec 2024-11-25T13:00:14,783 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T13:00:14,783 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 065eb16105c4aedcbfa7fb1ec2b14c86:B 2024-11-25T13:00:14,783 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-25T13:00:14,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,784 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61935 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-25T13:00:14,784 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): 065eb16105c4aedcbfa7fb1ec2b14c86/C is initiating minor compaction (all files) 2024-11-25T13:00:14,785 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 065eb16105c4aedcbfa7fb1ec2b14c86/C in TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:14,785 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/646aae351bbd4c54b84aa6a6e04db69e, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/b66cc7274d494fd2b929d6752b58d249, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/ba2f61c5cc914c73ada04671ae67aab3, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/33eeac8f284f49d88cda8d3bae9a7fd2, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/e43622462eab46a886ac24aea6041446] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp, totalSize=60.5 K 2024-11-25T13:00:14,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,785 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 646aae351bbd4c54b84aa6a6e04db69e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732539611334 2024-11-25T13:00:14,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,785 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting b66cc7274d494fd2b929d6752b58d249, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732539612054 2024-11-25T13:00:14,786 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting ba2f61c5cc914c73ada04671ae67aab3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1732539612717 2024-11-25T13:00:14,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,787 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 33eeac8f284f49d88cda8d3bae9a7fd2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1732539613425 2024-11-25T13:00:14,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,788 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting e43622462eab46a886ac24aea6041446, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1732539613558 2024-11-25T13:00:14,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:14,791 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T13:00:14,791 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 065eb16105c4aedcbfa7fb1ec2b14c86 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-25T13:00:14,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,812 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 065eb16105c4aedcbfa7fb1ec2b14c86#A#compaction#528 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T13:00:14,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,812 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/e417e9cf7d4d48b99c5359ebf406c6b6 is 175, key is test_row_0/A:col10/1732539613429/Put/seqid=0 2024-11-25T13:00:14,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,822 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 065eb16105c4aedcbfa7fb1ec2b14c86#C#compaction#532 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T13:00:14,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,822 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/6e1cbfee3adf4626b9e94903ccfb0754 is 50, key is test_row_0/C:col10/1732539613558/Put/seqid=0 2024-11-25T13:00:14,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411250a1f611364c24c81adeed21b716e1d66_065eb16105c4aedcbfa7fb1ec2b14c86 is 50, key is test_row_0/A:col10/1732539614790/Put/seqid=0 2024-11-25T13:00:14,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742461_1637 (size=31971) 2024-11-25T13:00:14,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-25T13:00:14,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742462_1638 (size=13051) 2024-11-25T13:00:14,857 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/6e1cbfee3adf4626b9e94903ccfb0754 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/6e1cbfee3adf4626b9e94903ccfb0754 2024-11-25T13:00:14,862 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 065eb16105c4aedcbfa7fb1ec2b14c86/C of 065eb16105c4aedcbfa7fb1ec2b14c86 into 6e1cbfee3adf4626b9e94903ccfb0754(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T13:00:14,862 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 065eb16105c4aedcbfa7fb1ec2b14c86: 2024-11-25T13:00:14,863 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., storeName=065eb16105c4aedcbfa7fb1ec2b14c86/C, priority=11, startTime=1732539614270; duration=0sec 2024-11-25T13:00:14,863 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T13:00:14,863 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 065eb16105c4aedcbfa7fb1ec2b14c86:C 2024-11-25T13:00:14,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742463_1639 (size=22618) 2024-11-25T13:00:14,879 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:14,886 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411250a1f611364c24c81adeed21b716e1d66_065eb16105c4aedcbfa7fb1ec2b14c86 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411250a1f611364c24c81adeed21b716e1d66_065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:14,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:14,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33782 deadline: 1732539674878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:14,889 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:14,889 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/8d63686e01de4235a4e6d1acacb29e14, store: [table=TestAcidGuarantees family=A region=065eb16105c4aedcbfa7fb1ec2b14c86] 2024-11-25T13:00:14,889 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:14,889 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:14,890 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/8d63686e01de4235a4e6d1acacb29e14 is 175, key is test_row_0/A:col10/1732539614790/Put/seqid=0 2024-11-25T13:00:14,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:14,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33764 deadline: 1732539674885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:14,895 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:14,895 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:14,895 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:14,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:14,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1732539674889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:14,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:14,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:14,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:14,903 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-25T13:00:14,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:14,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. as already flushing 2024-11-25T13:00:14,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:14,904 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:14,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:14,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:14,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742464_1640 (size=66023) 2024-11-25T13:00:14,920 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=342, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/8d63686e01de4235a4e6d1acacb29e14 2024-11-25T13:00:14,945 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/b50ff63861e5482e8ad93e7efe1a4b9e is 50, key is test_row_0/B:col10/1732539614790/Put/seqid=0 2024-11-25T13:00:14,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742465_1641 (size=12301) 2024-11-25T13:00:15,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:15,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33782 deadline: 1732539675000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:15,010 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:15,010 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:15,010 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:15,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:15,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33764 deadline: 1732539675006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:15,011 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:15,011 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:15,011 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:15,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:15,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1732539675009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:15,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:15,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:15,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:15,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-25T13:00:15,066 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-25T13:00:15,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:15,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. as already flushing 2024-11-25T13:00:15,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:15,067 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:15,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:15,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:15,222 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-25T13:00:15,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:15,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. as already flushing 2024-11-25T13:00:15,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:15,223 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:15,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:15,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:15,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:15,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33764 deadline: 1732539675228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:15,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:15,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33782 deadline: 1732539675228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:15,231 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:15,231 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:15,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:15,231 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:15,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1732539675228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:15,231 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:15,231 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:15,231 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:15,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:15,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:15,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:15,235 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/e417e9cf7d4d48b99c5359ebf406c6b6 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/e417e9cf7d4d48b99c5359ebf406c6b6 2024-11-25T13:00:15,262 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 065eb16105c4aedcbfa7fb1ec2b14c86/A of 065eb16105c4aedcbfa7fb1ec2b14c86 into e417e9cf7d4d48b99c5359ebf406c6b6(size=31.2 K), total size for store is 61.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T13:00:15,263 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 065eb16105c4aedcbfa7fb1ec2b14c86: 2024-11-25T13:00:15,263 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., storeName=065eb16105c4aedcbfa7fb1ec2b14c86/A, priority=12, startTime=1732539614268; duration=0sec 2024-11-25T13:00:15,263 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T13:00:15,263 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 065eb16105c4aedcbfa7fb1ec2b14c86:A 2024-11-25T13:00:15,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-25T13:00:15,377 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-25T13:00:15,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:15,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. as already flushing 2024-11-25T13:00:15,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:15,379 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:15,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:15,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:15,396 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/b50ff63861e5482e8ad93e7efe1a4b9e 2024-11-25T13:00:15,428 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/7df74bdf219c4e16b1ebd36383f64716 is 50, key is test_row_0/C:col10/1732539614790/Put/seqid=0 2024-11-25T13:00:15,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742466_1642 (size=12301) 2024-11-25T13:00:15,475 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/7df74bdf219c4e16b1ebd36383f64716 2024-11-25T13:00:15,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/8d63686e01de4235a4e6d1acacb29e14 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/8d63686e01de4235a4e6d1acacb29e14 2024-11-25T13:00:15,527 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/8d63686e01de4235a4e6d1acacb29e14, entries=350, sequenceid=342, filesize=64.5 K 2024-11-25T13:00:15,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/b50ff63861e5482e8ad93e7efe1a4b9e as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/b50ff63861e5482e8ad93e7efe1a4b9e 2024-11-25T13:00:15,533 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-25T13:00:15,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:15,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. as already flushing 2024-11-25T13:00:15,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:15,534 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:15,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:15,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:15,544 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/b50ff63861e5482e8ad93e7efe1a4b9e, entries=150, sequenceid=342, filesize=12.0 K 2024-11-25T13:00:15,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:15,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33764 deadline: 1732539675541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:15,546 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:15,546 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:15,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:15,546 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:15,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33782 deadline: 1732539675542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:15,547 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:15,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:15,547 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:15,547 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:15,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1732539675542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:15,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:15,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:15,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:15,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/7df74bdf219c4e16b1ebd36383f64716 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/7df74bdf219c4e16b1ebd36383f64716 2024-11-25T13:00:15,573 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/7df74bdf219c4e16b1ebd36383f64716, entries=150, sequenceid=342, filesize=12.0 K 2024-11-25T13:00:15,574 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 065eb16105c4aedcbfa7fb1ec2b14c86 in 783ms, sequenceid=342, compaction requested=true 2024-11-25T13:00:15,574 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 065eb16105c4aedcbfa7fb1ec2b14c86: 2024-11-25T13:00:15,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 065eb16105c4aedcbfa7fb1ec2b14c86:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T13:00:15,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T13:00:15,574 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T13:00:15,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 065eb16105c4aedcbfa7fb1ec2b14c86:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T13:00:15,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T13:00:15,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 065eb16105c4aedcbfa7fb1ec2b14c86:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T13:00:15,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-25T13:00:15,574 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T13:00:15,581 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 129249 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T13:00:15,581 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1541): 065eb16105c4aedcbfa7fb1ec2b14c86/A is initiating minor compaction (all files) 2024-11-25T13:00:15,582 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 065eb16105c4aedcbfa7fb1ec2b14c86/A in TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:15,582 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/e417e9cf7d4d48b99c5359ebf406c6b6, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/0af3647a9f194c8d8acabadf61612cd6, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/8d63686e01de4235a4e6d1acacb29e14] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp, totalSize=126.2 K 2024-11-25T13:00:15,582 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:15,582 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. files: [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/e417e9cf7d4d48b99c5359ebf406c6b6, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/0af3647a9f194c8d8acabadf61612cd6, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/8d63686e01de4235a4e6d1acacb29e14] 2024-11-25T13:00:15,582 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T13:00:15,582 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): 065eb16105c4aedcbfa7fb1ec2b14c86/B is initiating minor compaction (all files) 2024-11-25T13:00:15,582 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 065eb16105c4aedcbfa7fb1ec2b14c86/B in TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:15,582 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/e797a4811a274412bf15f54ce9a3d1ee, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/d8c71ab023c440c7bde170e89573dbbb, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/b50ff63861e5482e8ad93e7efe1a4b9e] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp, totalSize=36.7 K 2024-11-25T13:00:15,583 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting e797a4811a274412bf15f54ce9a3d1ee, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1732539613425 2024-11-25T13:00:15,583 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting e417e9cf7d4d48b99c5359ebf406c6b6, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1732539613425 2024-11-25T13:00:15,583 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting d8c71ab023c440c7bde170e89573dbbb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1732539613558 2024-11-25T13:00:15,583 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0af3647a9f194c8d8acabadf61612cd6, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1732539613558 2024-11-25T13:00:15,584 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting b50ff63861e5482e8ad93e7efe1a4b9e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1732539614753 2024-11-25T13:00:15,584 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d63686e01de4235a4e6d1acacb29e14, keycount=350, bloomtype=ROW, size=64.5 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1732539614753 2024-11-25T13:00:15,612 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 065eb16105c4aedcbfa7fb1ec2b14c86#B#compaction#536 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T13:00:15,613 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/d173f591b4b44eb2bf29021c430a74fe is 50, key is test_row_0/B:col10/1732539614790/Put/seqid=0 2024-11-25T13:00:15,624 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=065eb16105c4aedcbfa7fb1ec2b14c86] 2024-11-25T13:00:15,651 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241125403da7a3f4be494d9607f8083addc599_065eb16105c4aedcbfa7fb1ec2b14c86 store=[table=TestAcidGuarantees family=A region=065eb16105c4aedcbfa7fb1ec2b14c86] 2024-11-25T13:00:15,653 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241125403da7a3f4be494d9607f8083addc599_065eb16105c4aedcbfa7fb1ec2b14c86, store=[table=TestAcidGuarantees family=A region=065eb16105c4aedcbfa7fb1ec2b14c86] 2024-11-25T13:00:15,654 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125403da7a3f4be494d9607f8083addc599_065eb16105c4aedcbfa7fb1ec2b14c86 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=065eb16105c4aedcbfa7fb1ec2b14c86] 2024-11-25T13:00:15,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742467_1643 (size=13119) 2024-11-25T13:00:15,675 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/d173f591b4b44eb2bf29021c430a74fe as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/d173f591b4b44eb2bf29021c430a74fe 2024-11-25T13:00:15,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742468_1644 (size=4469) 2024-11-25T13:00:15,690 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 065eb16105c4aedcbfa7fb1ec2b14c86/B of 065eb16105c4aedcbfa7fb1ec2b14c86 into d173f591b4b44eb2bf29021c430a74fe(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T13:00:15,690 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 065eb16105c4aedcbfa7fb1ec2b14c86: 2024-11-25T13:00:15,690 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., storeName=065eb16105c4aedcbfa7fb1ec2b14c86/B, priority=13, startTime=1732539615574; duration=0sec 2024-11-25T13:00:15,690 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T13:00:15,690 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 065eb16105c4aedcbfa7fb1ec2b14c86:B 2024-11-25T13:00:15,690 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-25T13:00:15,691 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-25T13:00:15,691 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-25T13:00:15,691 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. because compaction request was cancelled 2024-11-25T13:00:15,691 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 065eb16105c4aedcbfa7fb1ec2b14c86:C 2024-11-25T13:00:15,692 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 065eb16105c4aedcbfa7fb1ec2b14c86#A#compaction#537 average throughput is 0.37 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T13:00:15,692 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/e11998640e9d4797a380116020fbd101 is 175, key is test_row_0/A:col10/1732539614790/Put/seqid=0 2024-11-25T13:00:15,692 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-25T13:00:15,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:15,693 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 065eb16105c4aedcbfa7fb1ec2b14c86 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-25T13:00:15,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742469_1645 (size=32073) 2024-11-25T13:00:15,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125732a9c8ca7594db99348f8a995a70382_065eb16105c4aedcbfa7fb1ec2b14c86 is 50, key is test_row_0/A:col10/1732539614887/Put/seqid=0 2024-11-25T13:00:15,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742470_1646 (size=12454) 2024-11-25T13:00:15,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:15,783 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125732a9c8ca7594db99348f8a995a70382_065eb16105c4aedcbfa7fb1ec2b14c86 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125732a9c8ca7594db99348f8a995a70382_065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:15,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/4572d0dfd4b54300b33afdabe7a22eac, store: [table=TestAcidGuarantees family=A region=065eb16105c4aedcbfa7fb1ec2b14c86] 2024-11-25T13:00:15,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/4572d0dfd4b54300b33afdabe7a22eac is 175, key is test_row_0/A:col10/1732539614887/Put/seqid=0 2024-11-25T13:00:15,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742471_1647 (size=31255) 2024-11-25T13:00:15,820 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=370, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/4572d0dfd4b54300b33afdabe7a22eac 2024-11-25T13:00:15,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/3adc37aad8ee4b4cb13a637c2684de06 is 50, key is test_row_0/B:col10/1732539614887/Put/seqid=0 2024-11-25T13:00:15,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742472_1648 (size=12301) 2024-11-25T13:00:15,851 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/3adc37aad8ee4b4cb13a637c2684de06 2024-11-25T13:00:15,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-25T13:00:15,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/38b2934fc17a4928a610ee204a6f5106 is 50, key is test_row_0/C:col10/1732539614887/Put/seqid=0 2024-11-25T13:00:15,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742473_1649 (size=12301) 2024-11-25T13:00:16,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. as already flushing 2024-11-25T13:00:16,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:16,094 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:16,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33764 deadline: 1732539676088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:16,099 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:16,099 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:16,099 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:16,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:16,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1732539676095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:16,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:16,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:16,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33782 deadline: 1732539676099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:16,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:16,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:16,108 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:16,108 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:16,108 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:16,150 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/e11998640e9d4797a380116020fbd101 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/e11998640e9d4797a380116020fbd101 2024-11-25T13:00:16,154 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 065eb16105c4aedcbfa7fb1ec2b14c86/A of 065eb16105c4aedcbfa7fb1ec2b14c86 into e11998640e9d4797a380116020fbd101(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T13:00:16,154 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 065eb16105c4aedcbfa7fb1ec2b14c86: 2024-11-25T13:00:16,154 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., storeName=065eb16105c4aedcbfa7fb1ec2b14c86/A, priority=13, startTime=1732539615574; duration=0sec 2024-11-25T13:00:16,154 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T13:00:16,154 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 065eb16105c4aedcbfa7fb1ec2b14c86:A 2024-11-25T13:00:16,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:16,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33764 deadline: 1732539676209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:16,216 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:16,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:16,216 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:16,216 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:16,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1732539676210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:16,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:16,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:16,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:16,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:16,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33782 deadline: 1732539676210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:16,221 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:16,221 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:16,221 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:16,290 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/38b2934fc17a4928a610ee204a6f5106 2024-11-25T13:00:16,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/4572d0dfd4b54300b33afdabe7a22eac as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/4572d0dfd4b54300b33afdabe7a22eac 2024-11-25T13:00:16,299 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/4572d0dfd4b54300b33afdabe7a22eac, entries=150, sequenceid=370, filesize=30.5 K 2024-11-25T13:00:16,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/3adc37aad8ee4b4cb13a637c2684de06 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/3adc37aad8ee4b4cb13a637c2684de06 2024-11-25T13:00:16,305 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/3adc37aad8ee4b4cb13a637c2684de06, entries=150, sequenceid=370, filesize=12.0 K 2024-11-25T13:00:16,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/38b2934fc17a4928a610ee204a6f5106 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/38b2934fc17a4928a610ee204a6f5106 2024-11-25T13:00:16,309 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/38b2934fc17a4928a610ee204a6f5106, entries=150, sequenceid=370, filesize=12.0 K 2024-11-25T13:00:16,310 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 065eb16105c4aedcbfa7fb1ec2b14c86 in 617ms, sequenceid=370, compaction requested=true 2024-11-25T13:00:16,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 065eb16105c4aedcbfa7fb1ec2b14c86: 2024-11-25T13:00:16,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:16,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-25T13:00:16,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-25T13:00:16,314 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-25T13:00:16,314 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5790 sec 2024-11-25T13:00:16,316 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 1.5840 sec 2024-11-25T13:00:16,423 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T13:00:16,423 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 065eb16105c4aedcbfa7fb1ec2b14c86 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-25T13:00:16,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:16,466 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125ee23a9e77ae0449a897f1c4554d432a6_065eb16105c4aedcbfa7fb1ec2b14c86 is 50, key is test_row_0/A:col10/1732539616422/Put/seqid=0 2024-11-25T13:00:16,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742474_1650 (size=14994) 2024-11-25T13:00:16,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:16,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33764 deadline: 1732539676515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:16,518 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:16,518 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:16,518 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:16,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:16,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1732539676517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:16,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:16,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:16,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:16,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:16,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33782 deadline: 1732539676517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:16,529 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:16,529 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:16,529 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:16,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:16,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33764 deadline: 1732539676630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:16,632 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:16,632 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:16,632 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:16,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:16,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:16,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33782 deadline: 1732539676640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:16,653 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:16,653 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:16,653 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:16,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1732539676630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:16,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:16,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:16,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:16,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:16,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33740 deadline: 1732539676650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:16,660 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 1ff8009e463e,35905,1732539449246 for 'test_row_1' in 065eb16105c4aedcbfa7fb1ec2b14c86 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8239 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T13:00:16,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:16,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:16,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:16,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:16,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33764 deadline: 1732539676840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:16,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:16,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:16,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:16,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:16,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33782 deadline: 1732539676860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:16,864 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:16,864 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:16,864 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:16,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:16,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1732539676869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:16,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:16,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:16,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:16,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-25T13:00:16,891 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T13:00:16,891 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T13:00:16,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-25T13:00:16,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-25T13:00:16,893 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T13:00:16,893 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T13:00:16,894 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T13:00:16,896 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:16,906 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125ee23a9e77ae0449a897f1c4554d432a6_065eb16105c4aedcbfa7fb1ec2b14c86 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125ee23a9e77ae0449a897f1c4554d432a6_065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:16,906 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/0637c5ff8ea047438721477de61f580b, store: [table=TestAcidGuarantees family=A region=065eb16105c4aedcbfa7fb1ec2b14c86] 2024-11-25T13:00:16,907 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/0637c5ff8ea047438721477de61f580b is 175, key is test_row_0/A:col10/1732539616422/Put/seqid=0 2024-11-25T13:00:16,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742475_1651 (size=39949) 2024-11-25T13:00:16,941 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=383, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/0637c5ff8ea047438721477de61f580b 2024-11-25T13:00:16,953 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/3f13d2e1d267489d900c464ca8f9dd35 is 50, key is test_row_0/B:col10/1732539616422/Put/seqid=0 2024-11-25T13:00:16,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742476_1652 (size=12301) 2024-11-25T13:00:16,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-25T13:00:17,046 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-25T13:00:17,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:17,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. as already flushing 2024-11-25T13:00:17,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:17,046 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:17,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:17,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:17,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:17,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33764 deadline: 1732539677150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:17,153 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:17,153 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:17,153 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:17,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:17,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33782 deadline: 1732539677171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:17,176 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:17,176 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:17,176 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:17,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:17,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1732539677179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:17,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:17,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:17,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:17,199 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-25T13:00:17,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:17,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. as already flushing 2024-11-25T13:00:17,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:17,199 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:17,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:17,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:17,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-25T13:00:17,352 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-25T13:00:17,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:17,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. as already flushing 2024-11-25T13:00:17,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:17,352 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:17,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:17,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:17,396 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/3f13d2e1d267489d900c464ca8f9dd35 2024-11-25T13:00:17,457 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/a1ae7e8c93774652a12525feba255ac2 is 50, key is test_row_0/C:col10/1732539616422/Put/seqid=0 2024-11-25T13:00:17,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742477_1653 (size=12301) 2024-11-25T13:00:17,500 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/a1ae7e8c93774652a12525feba255ac2 2024-11-25T13:00:17,504 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/0637c5ff8ea047438721477de61f580b as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/0637c5ff8ea047438721477de61f580b 2024-11-25T13:00:17,505 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-25T13:00:17,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:17,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. as already flushing 2024-11-25T13:00:17,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:17,505 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:17,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:17,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:17,509 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/0637c5ff8ea047438721477de61f580b, entries=200, sequenceid=383, filesize=39.0 K 2024-11-25T13:00:17,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/3f13d2e1d267489d900c464ca8f9dd35 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/3f13d2e1d267489d900c464ca8f9dd35 2024-11-25T13:00:17,515 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/3f13d2e1d267489d900c464ca8f9dd35, entries=150, sequenceid=383, filesize=12.0 K 2024-11-25T13:00:17,517 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/a1ae7e8c93774652a12525feba255ac2 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/a1ae7e8c93774652a12525feba255ac2 2024-11-25T13:00:17,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-25T13:00:17,521 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/a1ae7e8c93774652a12525feba255ac2, entries=150, sequenceid=383, filesize=12.0 K 2024-11-25T13:00:17,522 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 065eb16105c4aedcbfa7fb1ec2b14c86 in 1098ms, sequenceid=383, compaction requested=true 2024-11-25T13:00:17,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 065eb16105c4aedcbfa7fb1ec2b14c86: 2024-11-25T13:00:17,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 065eb16105c4aedcbfa7fb1ec2b14c86:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T13:00:17,522 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T13:00:17,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T13:00:17,522 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T13:00:17,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 065eb16105c4aedcbfa7fb1ec2b14c86:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T13:00:17,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T13:00:17,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 065eb16105c4aedcbfa7fb1ec2b14c86:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T13:00:17,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T13:00:17,523 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T13:00:17,523 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103277 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T13:00:17,523 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): 065eb16105c4aedcbfa7fb1ec2b14c86/B is initiating minor compaction (all files) 2024-11-25T13:00:17,523 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1541): 065eb16105c4aedcbfa7fb1ec2b14c86/A is initiating minor compaction (all files) 2024-11-25T13:00:17,523 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 065eb16105c4aedcbfa7fb1ec2b14c86/B in TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:17,523 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/d173f591b4b44eb2bf29021c430a74fe, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/3adc37aad8ee4b4cb13a637c2684de06, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/3f13d2e1d267489d900c464ca8f9dd35] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp, totalSize=36.8 K 2024-11-25T13:00:17,524 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 065eb16105c4aedcbfa7fb1ec2b14c86/A in TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:17,524 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/e11998640e9d4797a380116020fbd101, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/4572d0dfd4b54300b33afdabe7a22eac, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/0637c5ff8ea047438721477de61f580b] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp, totalSize=100.9 K 2024-11-25T13:00:17,524 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:17,524 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. files: [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/e11998640e9d4797a380116020fbd101, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/4572d0dfd4b54300b33afdabe7a22eac, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/0637c5ff8ea047438721477de61f580b] 2024-11-25T13:00:17,524 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting d173f591b4b44eb2bf29021c430a74fe, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1732539614753 2024-11-25T13:00:17,524 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting e11998640e9d4797a380116020fbd101, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1732539614753 2024-11-25T13:00:17,524 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 3adc37aad8ee4b4cb13a637c2684de06, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1732539614874 2024-11-25T13:00:17,525 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4572d0dfd4b54300b33afdabe7a22eac, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1732539614874 2024-11-25T13:00:17,525 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f13d2e1d267489d900c464ca8f9dd35, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1732539616083 2024-11-25T13:00:17,525 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0637c5ff8ea047438721477de61f580b, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1732539616083 2024-11-25T13:00:17,562 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 065eb16105c4aedcbfa7fb1ec2b14c86#B#compaction#544 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T13:00:17,562 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/c5202bf19a014d1280d357593232e9b1 is 50, key is test_row_0/B:col10/1732539616422/Put/seqid=0 2024-11-25T13:00:17,569 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=065eb16105c4aedcbfa7fb1ec2b14c86] 2024-11-25T13:00:17,607 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241125348e2fe20bf6449aa44f3aa24bb9fd87_065eb16105c4aedcbfa7fb1ec2b14c86 store=[table=TestAcidGuarantees family=A region=065eb16105c4aedcbfa7fb1ec2b14c86] 2024-11-25T13:00:17,609 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241125348e2fe20bf6449aa44f3aa24bb9fd87_065eb16105c4aedcbfa7fb1ec2b14c86, store=[table=TestAcidGuarantees family=A region=065eb16105c4aedcbfa7fb1ec2b14c86] 2024-11-25T13:00:17,609 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125348e2fe20bf6449aa44f3aa24bb9fd87_065eb16105c4aedcbfa7fb1ec2b14c86 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=065eb16105c4aedcbfa7fb1ec2b14c86] 2024-11-25T13:00:17,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742478_1654 (size=13221) 2024-11-25T13:00:17,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742479_1655 (size=4469) 2024-11-25T13:00:17,658 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-25T13:00:17,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:17,659 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 065eb16105c4aedcbfa7fb1ec2b14c86 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-25T13:00:17,673 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. as already flushing 2024-11-25T13:00:17,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:17,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411259a9472b94dcb467e99fae9928c23e4d0_065eb16105c4aedcbfa7fb1ec2b14c86 is 50, key is test_row_0/A:col10/1732539616516/Put/seqid=0 2024-11-25T13:00:17,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:17,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33764 deadline: 1732539677707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:17,714 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:17,714 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:17,714 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:17,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:17,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1732539677712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:17,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:17,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:17,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:17,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:17,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33782 deadline: 1732539677713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:17,715 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:17,715 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:17,716 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:17,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742480_1656 (size=12454) 2024-11-25T13:00:17,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:17,746 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411259a9472b94dcb467e99fae9928c23e4d0_065eb16105c4aedcbfa7fb1ec2b14c86 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411259a9472b94dcb467e99fae9928c23e4d0_065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:17,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/93aa332746564b6c8c96da12ab91d882, store: [table=TestAcidGuarantees family=A region=065eb16105c4aedcbfa7fb1ec2b14c86] 2024-11-25T13:00:17,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/93aa332746564b6c8c96da12ab91d882 is 175, key is test_row_0/A:col10/1732539616516/Put/seqid=0 2024-11-25T13:00:17,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742481_1657 (size=31255) 2024-11-25T13:00:17,794 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=407, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/93aa332746564b6c8c96da12ab91d882 2024-11-25T13:00:17,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/584e3c4a1a914930bf833950fb8f3fa8 is 50, key is test_row_0/B:col10/1732539616516/Put/seqid=0 2024-11-25T13:00:17,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:17,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:17,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33782 deadline: 1732539677823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:17,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33764 deadline: 1732539677823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:17,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:17,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1732539677823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:17,829 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:17,829 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:17,829 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:17,830 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:17,830 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:17,830 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:17,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:17,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:17,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:17,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742482_1658 (size=12301) 2024-11-25T13:00:17,886 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/584e3c4a1a914930bf833950fb8f3fa8 2024-11-25T13:00:17,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/aa6b3b9e20f64a5797cdfbe9f2b1cc2b is 50, key is test_row_0/C:col10/1732539616516/Put/seqid=0 2024-11-25T13:00:17,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742483_1659 (size=12301) 2024-11-25T13:00:18,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-25T13:00:18,046 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/c5202bf19a014d1280d357593232e9b1 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/c5202bf19a014d1280d357593232e9b1 2024-11-25T13:00:18,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:18,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33764 deadline: 1732539678047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:18,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:18,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33782 deadline: 1732539678047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:18,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:18,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1732539678047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:18,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:18,058 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:18,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:18,059 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:18,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:18,059 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:18,059 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:18,059 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:18,059 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:18,062 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 065eb16105c4aedcbfa7fb1ec2b14c86#A#compaction#545 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T13:00:18,062 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/5b5171ebe1d8486a908063d9e37d207b is 175, key is test_row_0/A:col10/1732539616422/Put/seqid=0 2024-11-25T13:00:18,068 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 065eb16105c4aedcbfa7fb1ec2b14c86/B of 065eb16105c4aedcbfa7fb1ec2b14c86 into c5202bf19a014d1280d357593232e9b1(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T13:00:18,068 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 065eb16105c4aedcbfa7fb1ec2b14c86: 2024-11-25T13:00:18,068 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., storeName=065eb16105c4aedcbfa7fb1ec2b14c86/B, priority=13, startTime=1732539617522; duration=0sec 2024-11-25T13:00:18,068 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T13:00:18,068 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 065eb16105c4aedcbfa7fb1ec2b14c86:B 2024-11-25T13:00:18,068 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T13:00:18,079 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T13:00:18,079 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): 065eb16105c4aedcbfa7fb1ec2b14c86/C is initiating minor compaction (all files) 2024-11-25T13:00:18,080 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 065eb16105c4aedcbfa7fb1ec2b14c86/C in TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:18,080 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/6e1cbfee3adf4626b9e94903ccfb0754, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/7df74bdf219c4e16b1ebd36383f64716, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/38b2934fc17a4928a610ee204a6f5106, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/a1ae7e8c93774652a12525feba255ac2] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp, totalSize=48.8 K 2024-11-25T13:00:18,080 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e1cbfee3adf4626b9e94903ccfb0754, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1732539613558 2024-11-25T13:00:18,080 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 7df74bdf219c4e16b1ebd36383f64716, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1732539614753 2024-11-25T13:00:18,081 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 38b2934fc17a4928a610ee204a6f5106, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1732539614874 2024-11-25T13:00:18,084 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting a1ae7e8c93774652a12525feba255ac2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1732539616083 2024-11-25T13:00:18,118 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 065eb16105c4aedcbfa7fb1ec2b14c86#C#compaction#549 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T13:00:18,118 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/a6123c68353a4d0099f9619344da39cd is 50, key is test_row_0/C:col10/1732539616422/Put/seqid=0 2024-11-25T13:00:18,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742484_1660 (size=32175) 2024-11-25T13:00:18,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742485_1661 (size=13187) 2024-11-25T13:00:18,372 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/aa6b3b9e20f64a5797cdfbe9f2b1cc2b 2024-11-25T13:00:18,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:18,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1732539678370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:18,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:18,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:18,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:18,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:18,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33764 deadline: 1732539678370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:18,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:18,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:18,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33782 deadline: 1732539678370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:18,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:18,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:18,375 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:18,375 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:18,375 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:18,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/93aa332746564b6c8c96da12ab91d882 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/93aa332746564b6c8c96da12ab91d882 2024-11-25T13:00:18,379 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/93aa332746564b6c8c96da12ab91d882, entries=150, sequenceid=407, filesize=30.5 K 2024-11-25T13:00:18,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/584e3c4a1a914930bf833950fb8f3fa8 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/584e3c4a1a914930bf833950fb8f3fa8 2024-11-25T13:00:18,384 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/584e3c4a1a914930bf833950fb8f3fa8, entries=150, sequenceid=407, filesize=12.0 K 2024-11-25T13:00:18,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/aa6b3b9e20f64a5797cdfbe9f2b1cc2b as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/aa6b3b9e20f64a5797cdfbe9f2b1cc2b 2024-11-25T13:00:18,388 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/aa6b3b9e20f64a5797cdfbe9f2b1cc2b, entries=150, sequenceid=407, filesize=12.0 K 2024-11-25T13:00:18,390 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 065eb16105c4aedcbfa7fb1ec2b14c86 in 731ms, sequenceid=407, compaction requested=false 2024-11-25T13:00:18,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 065eb16105c4aedcbfa7fb1ec2b14c86: 2024-11-25T13:00:18,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:18,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-25T13:00:18,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-25T13:00:18,415 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-25T13:00:18,415 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5050 sec 2024-11-25T13:00:18,417 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.5250 sec 2024-11-25T13:00:18,582 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/5b5171ebe1d8486a908063d9e37d207b as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/5b5171ebe1d8486a908063d9e37d207b 2024-11-25T13:00:18,586 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 065eb16105c4aedcbfa7fb1ec2b14c86/A of 065eb16105c4aedcbfa7fb1ec2b14c86 into 5b5171ebe1d8486a908063d9e37d207b(size=31.4 K), total size for store is 61.9 K. This selection was in queue for 0sec, and took 1sec to execute. 2024-11-25T13:00:18,587 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 065eb16105c4aedcbfa7fb1ec2b14c86: 2024-11-25T13:00:18,587 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., storeName=065eb16105c4aedcbfa7fb1ec2b14c86/A, priority=13, startTime=1732539617522; duration=1sec 2024-11-25T13:00:18,587 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T13:00:18,587 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 065eb16105c4aedcbfa7fb1ec2b14c86:A 2024-11-25T13:00:18,602 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/a6123c68353a4d0099f9619344da39cd as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/a6123c68353a4d0099f9619344da39cd 2024-11-25T13:00:18,607 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 065eb16105c4aedcbfa7fb1ec2b14c86/C of 065eb16105c4aedcbfa7fb1ec2b14c86 into a6123c68353a4d0099f9619344da39cd(size=12.9 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T13:00:18,607 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 065eb16105c4aedcbfa7fb1ec2b14c86: 2024-11-25T13:00:18,607 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., storeName=065eb16105c4aedcbfa7fb1ec2b14c86/C, priority=12, startTime=1732539617522; duration=0sec 2024-11-25T13:00:18,607 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T13:00:18,607 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 065eb16105c4aedcbfa7fb1ec2b14c86:C 2024-11-25T13:00:18,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:18,887 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T13:00:18,887 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 065eb16105c4aedcbfa7fb1ec2b14c86 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-25T13:00:18,926 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411256f9fc12468ab40aabc6f774897674d33_065eb16105c4aedcbfa7fb1ec2b14c86 is 50, key is test_row_0/A:col10/1732539617688/Put/seqid=0 2024-11-25T13:00:18,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:18,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33764 deadline: 1732539678942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:18,950 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:18,950 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:18,950 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:18,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:18,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1732539678943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:18,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:18,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:18,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:18,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:18,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33782 deadline: 1732539678944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:18,951 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:18,952 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:18,952 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:18,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742486_1662 (size=12454) 2024-11-25T13:00:19,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-25T13:00:19,040 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T13:00:19,040 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T13:00:19,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-25T13:00:19,043 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T13:00:19,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-25T13:00:19,044 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T13:00:19,044 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T13:00:19,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:19,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33764 deadline: 1732539679060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:19,064 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:19,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:19,064 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:19,064 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:19,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1732539679060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:19,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:19,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33782 deadline: 1732539679060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:19,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:19,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:19,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:19,064 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:19,064 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:19,064 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:19,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-25T13:00:19,197 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-25T13:00:19,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:19,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. as already flushing 2024-11-25T13:00:19,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:19,198 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:19,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:19,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:19,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:19,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33764 deadline: 1732539679276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:19,277 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:19,277 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:19,277 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:19,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:19,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1732539679276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:19,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:19,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:19,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:19,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:19,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33782 deadline: 1732539679276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:19,283 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:19,283 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:19,283 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:19,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-25T13:00:19,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:19,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. as already flushing 2024-11-25T13:00:19,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:19,351 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:19,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:19,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:19,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-25T13:00:19,365 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:19,369 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411256f9fc12468ab40aabc6f774897674d33_065eb16105c4aedcbfa7fb1ec2b14c86 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411256f9fc12468ab40aabc6f774897674d33_065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:19,370 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/00c0ad8b01dc43649b7da48b28deef08, store: [table=TestAcidGuarantees family=A region=065eb16105c4aedcbfa7fb1ec2b14c86] 2024-11-25T13:00:19,371 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/00c0ad8b01dc43649b7da48b28deef08 is 175, key is test_row_0/A:col10/1732539617688/Put/seqid=0 2024-11-25T13:00:19,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742487_1663 (size=31255) 2024-11-25T13:00:19,428 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=425, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/00c0ad8b01dc43649b7da48b28deef08 2024-11-25T13:00:19,447 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/d3319f77a810494faf3247091d1acec1 is 50, key is test_row_0/B:col10/1732539617688/Put/seqid=0 2024-11-25T13:00:19,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742488_1664 (size=12301) 2024-11-25T13:00:19,508 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-25T13:00:19,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:19,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. as already flushing 2024-11-25T13:00:19,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:19,508 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:19,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:19,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:19,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:19,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1732539679580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:19,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:19,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:19,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33764 deadline: 1732539679580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:19,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:19,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:19,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:19,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:19,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:19,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:19,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33782 deadline: 1732539679590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:19,596 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:19,596 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:19,596 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:19,663 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-25T13:00:19,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:19,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. as already flushing 2024-11-25T13:00:19,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:19,663 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:19,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:19,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:19,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-25T13:00:19,818 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-25T13:00:19,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:19,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. as already flushing 2024-11-25T13:00:19,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:19,819 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:19,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:19,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:19,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:19,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33778 deadline: 1732539679860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:19,867 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 1ff8009e463e,35905,1732539449246 for 'test_row_2' in 065eb16105c4aedcbfa7fb1ec2b14c86 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18341 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T13:00:19,868 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:19,868 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:19,868 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:19,903 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=425 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/d3319f77a810494faf3247091d1acec1 2024-11-25T13:00:19,923 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/3a24aab432904317826b53dbdf6f4c39 is 50, key is test_row_0/C:col10/1732539617688/Put/seqid=0 2024-11-25T13:00:19,972 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-25T13:00:19,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:19,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. as already flushing 2024-11-25T13:00:19,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:19,972 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:19,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:19,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:19,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742489_1665 (size=12301) 2024-11-25T13:00:20,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:20,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1732539680091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:20,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:20,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33764 deadline: 1732539680091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:20,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:20,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:20,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:20,094 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:20,094 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:20,094 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:20,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:20,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33782 deadline: 1732539680110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:20,113 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:20,113 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:20,113 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:20,126 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-25T13:00:20,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:20,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. as already flushing 2024-11-25T13:00:20,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:20,126 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:20,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:20,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:20,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-25T13:00:20,279 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-25T13:00:20,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:20,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. as already flushing 2024-11-25T13:00:20,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:20,279 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:20,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:20,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T13:00:20,392 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=425 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/3a24aab432904317826b53dbdf6f4c39 2024-11-25T13:00:20,403 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/00c0ad8b01dc43649b7da48b28deef08 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/00c0ad8b01dc43649b7da48b28deef08 2024-11-25T13:00:20,414 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/00c0ad8b01dc43649b7da48b28deef08, entries=150, sequenceid=425, filesize=30.5 K 2024-11-25T13:00:20,415 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/d3319f77a810494faf3247091d1acec1 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/d3319f77a810494faf3247091d1acec1 2024-11-25T13:00:20,419 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/d3319f77a810494faf3247091d1acec1, entries=150, sequenceid=425, filesize=12.0 K 2024-11-25T13:00:20,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/3a24aab432904317826b53dbdf6f4c39 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/3a24aab432904317826b53dbdf6f4c39 2024-11-25T13:00:20,424 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/3a24aab432904317826b53dbdf6f4c39, entries=150, sequenceid=425, filesize=12.0 K 2024-11-25T13:00:20,425 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for 065eb16105c4aedcbfa7fb1ec2b14c86 in 1538ms, sequenceid=425, compaction requested=true 2024-11-25T13:00:20,425 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 065eb16105c4aedcbfa7fb1ec2b14c86: 2024-11-25T13:00:20,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 065eb16105c4aedcbfa7fb1ec2b14c86:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T13:00:20,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T13:00:20,426 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T13:00:20,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 065eb16105c4aedcbfa7fb1ec2b14c86:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T13:00:20,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T13:00:20,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 065eb16105c4aedcbfa7fb1ec2b14c86:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T13:00:20,426 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T13:00:20,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T13:00:20,426 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T13:00:20,426 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): 065eb16105c4aedcbfa7fb1ec2b14c86/B is initiating minor compaction (all files) 2024-11-25T13:00:20,426 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 065eb16105c4aedcbfa7fb1ec2b14c86/B in TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:20,427 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/c5202bf19a014d1280d357593232e9b1, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/584e3c4a1a914930bf833950fb8f3fa8, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/d3319f77a810494faf3247091d1acec1] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp, totalSize=36.9 K 2024-11-25T13:00:20,427 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94685 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T13:00:20,427 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting c5202bf19a014d1280d357593232e9b1, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1732539616083 2024-11-25T13:00:20,427 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1541): 065eb16105c4aedcbfa7fb1ec2b14c86/A is initiating minor compaction (all files) 2024-11-25T13:00:20,427 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 065eb16105c4aedcbfa7fb1ec2b14c86/A in TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:20,427 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/5b5171ebe1d8486a908063d9e37d207b, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/93aa332746564b6c8c96da12ab91d882, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/00c0ad8b01dc43649b7da48b28deef08] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp, totalSize=92.5 K 2024-11-25T13:00:20,427 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:20,427 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. files: [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/5b5171ebe1d8486a908063d9e37d207b, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/93aa332746564b6c8c96da12ab91d882, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/00c0ad8b01dc43649b7da48b28deef08] 2024-11-25T13:00:20,427 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 584e3c4a1a914930bf833950fb8f3fa8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1732539616506 2024-11-25T13:00:20,428 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b5171ebe1d8486a908063d9e37d207b, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1732539616083 2024-11-25T13:00:20,428 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93aa332746564b6c8c96da12ab91d882, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1732539616506 2024-11-25T13:00:20,428 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting d3319f77a810494faf3247091d1acec1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1732539617688 2024-11-25T13:00:20,429 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00c0ad8b01dc43649b7da48b28deef08, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1732539617688 2024-11-25T13:00:20,435 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 065eb16105c4aedcbfa7fb1ec2b14c86#B#compaction#553 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T13:00:20,435 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/2a5174cbfee142eb898f2de1a3d11284 is 50, key is test_row_0/B:col10/1732539617688/Put/seqid=0 2024-11-25T13:00:20,447 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=065eb16105c4aedcbfa7fb1ec2b14c86] 2024-11-25T13:00:20,450 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35905 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-25T13:00:20,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:20,451 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 065eb16105c4aedcbfa7fb1ec2b14c86 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-25T13:00:20,457 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411259407852ab08240728ead65c14747b5f4_065eb16105c4aedcbfa7fb1ec2b14c86 store=[table=TestAcidGuarantees family=A region=065eb16105c4aedcbfa7fb1ec2b14c86] 2024-11-25T13:00:20,459 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411259407852ab08240728ead65c14747b5f4_065eb16105c4aedcbfa7fb1ec2b14c86, store=[table=TestAcidGuarantees family=A region=065eb16105c4aedcbfa7fb1ec2b14c86] 2024-11-25T13:00:20,459 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411259407852ab08240728ead65c14747b5f4_065eb16105c4aedcbfa7fb1ec2b14c86 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=065eb16105c4aedcbfa7fb1ec2b14c86] 2024-11-25T13:00:20,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742490_1666 (size=13323) 2024-11-25T13:00:20,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112521ff85d72ba14b2b9b87b89e012941f6_065eb16105c4aedcbfa7fb1ec2b14c86 is 50, key is test_row_0/A:col10/1732539618939/Put/seqid=0 2024-11-25T13:00:20,502 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/2a5174cbfee142eb898f2de1a3d11284 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/2a5174cbfee142eb898f2de1a3d11284 2024-11-25T13:00:20,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742491_1667 (size=4469) 2024-11-25T13:00:20,521 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 065eb16105c4aedcbfa7fb1ec2b14c86#A#compaction#554 average throughput is 0.33 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T13:00:20,521 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/7df79d6fa00a40d2adfe9a7eea4d1ca2 is 175, key is test_row_0/A:col10/1732539617688/Put/seqid=0 2024-11-25T13:00:20,526 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 065eb16105c4aedcbfa7fb1ec2b14c86/B of 065eb16105c4aedcbfa7fb1ec2b14c86 into 2a5174cbfee142eb898f2de1a3d11284(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T13:00:20,526 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 065eb16105c4aedcbfa7fb1ec2b14c86: 2024-11-25T13:00:20,526 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., storeName=065eb16105c4aedcbfa7fb1ec2b14c86/B, priority=13, startTime=1732539620426; duration=0sec 2024-11-25T13:00:20,526 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T13:00:20,526 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 065eb16105c4aedcbfa7fb1ec2b14c86:B 2024-11-25T13:00:20,526 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T13:00:20,526 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T13:00:20,527 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): 065eb16105c4aedcbfa7fb1ec2b14c86/C is initiating minor compaction (all files) 2024-11-25T13:00:20,527 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 065eb16105c4aedcbfa7fb1ec2b14c86/C in TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:20,527 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/a6123c68353a4d0099f9619344da39cd, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/aa6b3b9e20f64a5797cdfbe9f2b1cc2b, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/3a24aab432904317826b53dbdf6f4c39] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp, totalSize=36.9 K 2024-11-25T13:00:20,527 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting a6123c68353a4d0099f9619344da39cd, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1732539616083 2024-11-25T13:00:20,531 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting aa6b3b9e20f64a5797cdfbe9f2b1cc2b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1732539616506 2024-11-25T13:00:20,532 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a24aab432904317826b53dbdf6f4c39, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1732539617688 2024-11-25T13:00:20,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742492_1668 (size=12454) 2024-11-25T13:00:20,552 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 065eb16105c4aedcbfa7fb1ec2b14c86#C#compaction#556 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T13:00:20,553 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/557de74cc5f946e286f119fff14867a7 is 50, key is test_row_0/C:col10/1732539617688/Put/seqid=0 2024-11-25T13:00:20,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742493_1669 (size=32277) 2024-11-25T13:00:20,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742494_1670 (size=13289) 2024-11-25T13:00:20,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:20,948 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112521ff85d72ba14b2b9b87b89e012941f6_065eb16105c4aedcbfa7fb1ec2b14c86 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112521ff85d72ba14b2b9b87b89e012941f6_065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:20,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/0f6bb67859bb4056a09638496ca2b0c1, store: [table=TestAcidGuarantees family=A region=065eb16105c4aedcbfa7fb1ec2b14c86] 2024-11-25T13:00:20,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/0f6bb67859bb4056a09638496ca2b0c1 is 175, key is test_row_0/A:col10/1732539618939/Put/seqid=0 2024-11-25T13:00:20,960 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/7df79d6fa00a40d2adfe9a7eea4d1ca2 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/7df79d6fa00a40d2adfe9a7eea4d1ca2 2024-11-25T13:00:20,964 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 065eb16105c4aedcbfa7fb1ec2b14c86/A of 065eb16105c4aedcbfa7fb1ec2b14c86 into 7df79d6fa00a40d2adfe9a7eea4d1ca2(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T13:00:20,964 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 065eb16105c4aedcbfa7fb1ec2b14c86: 2024-11-25T13:00:20,964 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., storeName=065eb16105c4aedcbfa7fb1ec2b14c86/A, priority=13, startTime=1732539620425; duration=0sec 2024-11-25T13:00:20,964 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T13:00:20,964 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 065eb16105c4aedcbfa7fb1ec2b14c86:A 2024-11-25T13:00:20,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742495_1671 (size=31255) 2024-11-25T13:00:20,973 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/557de74cc5f946e286f119fff14867a7 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/557de74cc5f946e286f119fff14867a7 2024-11-25T13:00:20,977 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 065eb16105c4aedcbfa7fb1ec2b14c86/C of 065eb16105c4aedcbfa7fb1ec2b14c86 into 557de74cc5f946e286f119fff14867a7(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T13:00:20,977 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 065eb16105c4aedcbfa7fb1ec2b14c86: 2024-11-25T13:00:20,977 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., storeName=065eb16105c4aedcbfa7fb1ec2b14c86/C, priority=13, startTime=1732539620426; duration=0sec 2024-11-25T13:00:20,977 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T13:00:20,977 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 065eb16105c4aedcbfa7fb1ec2b14c86:C 2024-11-25T13:00:20,978 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=447, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/0f6bb67859bb4056a09638496ca2b0c1 2024-11-25T13:00:20,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/09aed9bc5a914e1897ef4ced07e6a3f4 is 50, key is test_row_0/B:col10/1732539618939/Put/seqid=0 2024-11-25T13:00:21,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742496_1672 (size=12301) 2024-11-25T13:00:21,039 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=447 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/09aed9bc5a914e1897ef4ced07e6a3f4 2024-11-25T13:00:21,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/a908404ffd2049a594bc8e97847b5d3f is 50, key is test_row_0/C:col10/1732539618939/Put/seqid=0 2024-11-25T13:00:21,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742497_1673 (size=12301) 2024-11-25T13:00:21,104 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. as already flushing 2024-11-25T13:00:21,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:21,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:21,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1732539681137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:21,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:21,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33782 deadline: 1732539681138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:21,144 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:21,144 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:21,144 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:21,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:21,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:21,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:21,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:21,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33764 deadline: 1732539681143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:21,163 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:21,163 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:21,164 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:21,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-25T13:00:21,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:21,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1732539681250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:21,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:21,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:21,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:21,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:21,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33782 deadline: 1732539681250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:21,267 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:21,267 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:21,267 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:21,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:21,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33764 deadline: 1732539681270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:21,273 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:21,273 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:21,273 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:21,465 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=447 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/a908404ffd2049a594bc8e97847b5d3f 2024-11-25T13:00:21,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/0f6bb67859bb4056a09638496ca2b0c1 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/0f6bb67859bb4056a09638496ca2b0c1 2024-11-25T13:00:21,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:21,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1732539681470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:21,473 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/0f6bb67859bb4056a09638496ca2b0c1, entries=150, sequenceid=447, filesize=30.5 K 2024-11-25T13:00:21,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T13:00:21,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35905 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33782 deadline: 1732539681470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:21,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:21,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:21,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:21,473 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 , the old value is region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:21,474 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T13:00:21,474 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., hostname=1ff8009e463e,35905,1732539449246, seqNum=5 because the exception is null or not the one we care about 2024-11-25T13:00:21,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/09aed9bc5a914e1897ef4ced07e6a3f4 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/09aed9bc5a914e1897ef4ced07e6a3f4 2024-11-25T13:00:21,477 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/09aed9bc5a914e1897ef4ced07e6a3f4, entries=150, sequenceid=447, filesize=12.0 K 2024-11-25T13:00:21,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/a908404ffd2049a594bc8e97847b5d3f as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/a908404ffd2049a594bc8e97847b5d3f 2024-11-25T13:00:21,481 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/a908404ffd2049a594bc8e97847b5d3f, entries=150, sequenceid=447, filesize=12.0 K 2024-11-25T13:00:21,481 INFO [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 065eb16105c4aedcbfa7fb1ec2b14c86 in 1030ms, sequenceid=447, compaction requested=false 2024-11-25T13:00:21,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 065eb16105c4aedcbfa7fb1ec2b14c86: 2024-11-25T13:00:21,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:21,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-25T13:00:21,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-25T13:00:21,484 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-25T13:00:21,484 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4380 sec 2024-11-25T13:00:21,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:21,484 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T13:00:21,484 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 065eb16105c4aedcbfa7fb1ec2b14c86 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-25T13:00:21,486 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 2.4450 sec 2024-11-25T13:00:21,493 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125cc58a260b5504369a0b103c79fda0b99_065eb16105c4aedcbfa7fb1ec2b14c86 is 50, key is test_row_0/A:col10/1732539621136/Put/seqid=0 2024-11-25T13:00:21,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742498_1674 (size=14994) 2024-11-25T13:00:21,497 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:21,501 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125cc58a260b5504369a0b103c79fda0b99_065eb16105c4aedcbfa7fb1ec2b14c86 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125cc58a260b5504369a0b103c79fda0b99_065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:21,501 INFO [Thread-2447 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2447. 2024-11-25T13:00:21,501 DEBUG [Thread-2447 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T13:00:21,501 DEBUG [Thread-2447 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T13:00:21,502 DEBUG [Thread-2447 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T13:00:21,502 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T13:00:21,502 INFO [Thread-2451 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2451. 2024-11-25T13:00:21,502 DEBUG [Thread-2451 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T13:00:21,502 DEBUG [Thread-2451 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T13:00:21,502 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/3d14e3987e7b4026b59e6f1f2256bad4, store: [table=TestAcidGuarantees family=A region=065eb16105c4aedcbfa7fb1ec2b14c86] 2024-11-25T13:00:21,502 DEBUG [Thread-2451 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T13:00:21,502 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T13:00:21,502 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/3d14e3987e7b4026b59e6f1f2256bad4 is 175, key is test_row_0/A:col10/1732539621136/Put/seqid=0 2024-11-25T13:00:21,503 INFO [Thread-2453 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2453. 2024-11-25T13:00:21,503 DEBUG [Thread-2453 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T13:00:21,503 DEBUG [Thread-2453 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T13:00:21,503 DEBUG [Thread-2453 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T13:00:21,503 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T13:00:21,504 INFO [Thread-2449 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2449. 2024-11-25T13:00:21,504 DEBUG [Thread-2449 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T13:00:21,504 INFO [Thread-2440 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2440. 2024-11-25T13:00:21,504 DEBUG [Thread-2449 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T13:00:21,504 DEBUG [Thread-2440 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T13:00:21,504 DEBUG [Thread-2440 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T13:00:21,504 DEBUG [Thread-2449 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T13:00:21,504 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T13:00:21,504 DEBUG [Thread-2440 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T13:00:21,504 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T13:00:21,507 INFO [Thread-2445 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2445. 2024-11-25T13:00:21,507 DEBUG [Thread-2445 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T13:00:21,507 DEBUG [Thread-2445 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T13:00:21,507 DEBUG [Thread-2445 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T13:00:21,507 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T13:00:21,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742499_1675 (size=39949) 2024-11-25T13:00:21,781 INFO [Thread-2434 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2434. 2024-11-25T13:00:21,781 INFO [Thread-2438 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2438. 2024-11-25T13:00:21,781 DEBUG [Thread-2434 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T13:00:21,781 DEBUG [Thread-2438 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T13:00:21,781 DEBUG [Thread-2434 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T13:00:21,781 DEBUG [Thread-2438 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T13:00:21,781 DEBUG [Thread-2434 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T13:00:21,781 DEBUG [Thread-2438 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T13:00:21,781 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T13:00:21,781 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T13:00:21,908 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=465, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/3d14e3987e7b4026b59e6f1f2256bad4 2024-11-25T13:00:21,914 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/a62e40c9be9a48a58bb5ff8e3a7b7448 is 50, key is test_row_0/B:col10/1732539621136/Put/seqid=0 2024-11-25T13:00:21,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742500_1676 (size=12301) 2024-11-25T13:00:21,923 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/a62e40c9be9a48a58bb5ff8e3a7b7448 2024-11-25T13:00:21,932 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/8fc3d1eae65e4dd98de707b35c98e96f is 50, key is test_row_0/C:col10/1732539621136/Put/seqid=0 2024-11-25T13:00:21,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742501_1677 (size=12301) 2024-11-25T13:00:22,340 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/8fc3d1eae65e4dd98de707b35c98e96f 2024-11-25T13:00:22,348 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/3d14e3987e7b4026b59e6f1f2256bad4 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/3d14e3987e7b4026b59e6f1f2256bad4 2024-11-25T13:00:22,364 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/3d14e3987e7b4026b59e6f1f2256bad4, entries=200, sequenceid=465, filesize=39.0 K 2024-11-25T13:00:22,367 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/a62e40c9be9a48a58bb5ff8e3a7b7448 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/a62e40c9be9a48a58bb5ff8e3a7b7448 2024-11-25T13:00:22,378 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/a62e40c9be9a48a58bb5ff8e3a7b7448, entries=150, sequenceid=465, filesize=12.0 K 2024-11-25T13:00:22,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/8fc3d1eae65e4dd98de707b35c98e96f as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/8fc3d1eae65e4dd98de707b35c98e96f 2024-11-25T13:00:22,382 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/8fc3d1eae65e4dd98de707b35c98e96f, entries=150, sequenceid=465, filesize=12.0 K 2024-11-25T13:00:22,382 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=46.96 KB/48090 for 065eb16105c4aedcbfa7fb1ec2b14c86 in 898ms, sequenceid=465, compaction requested=true 2024-11-25T13:00:22,383 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 065eb16105c4aedcbfa7fb1ec2b14c86: 2024-11-25T13:00:22,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 065eb16105c4aedcbfa7fb1ec2b14c86:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T13:00:22,383 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T13:00:22,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T13:00:22,383 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T13:00:22,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 065eb16105c4aedcbfa7fb1ec2b14c86:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T13:00:22,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T13:00:22,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 065eb16105c4aedcbfa7fb1ec2b14c86:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T13:00:22,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T13:00:22,383 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103481 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T13:00:22,383 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T13:00:22,383 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1541): 065eb16105c4aedcbfa7fb1ec2b14c86/A is initiating minor compaction (all files) 2024-11-25T13:00:22,383 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): 065eb16105c4aedcbfa7fb1ec2b14c86/B is initiating minor compaction (all files) 2024-11-25T13:00:22,384 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 065eb16105c4aedcbfa7fb1ec2b14c86/A in TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:22,384 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 065eb16105c4aedcbfa7fb1ec2b14c86/B in TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:22,384 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/2a5174cbfee142eb898f2de1a3d11284, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/09aed9bc5a914e1897ef4ced07e6a3f4, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/a62e40c9be9a48a58bb5ff8e3a7b7448] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp, totalSize=37.0 K 2024-11-25T13:00:22,384 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/7df79d6fa00a40d2adfe9a7eea4d1ca2, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/0f6bb67859bb4056a09638496ca2b0c1, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/3d14e3987e7b4026b59e6f1f2256bad4] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp, totalSize=101.1 K 2024-11-25T13:00:22,384 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:22,384 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. files: [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/7df79d6fa00a40d2adfe9a7eea4d1ca2, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/0f6bb67859bb4056a09638496ca2b0c1, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/3d14e3987e7b4026b59e6f1f2256bad4] 2024-11-25T13:00:22,384 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a5174cbfee142eb898f2de1a3d11284, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1732539617688 2024-11-25T13:00:22,384 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7df79d6fa00a40d2adfe9a7eea4d1ca2, keycount=150, bloomtype=ROW, size=31.5 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1732539617688 2024-11-25T13:00:22,384 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 09aed9bc5a914e1897ef4ced07e6a3f4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1732539618930 2024-11-25T13:00:22,384 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f6bb67859bb4056a09638496ca2b0c1, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1732539618930 2024-11-25T13:00:22,384 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting a62e40c9be9a48a58bb5ff8e3a7b7448, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1732539621136 2024-11-25T13:00:22,384 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d14e3987e7b4026b59e6f1f2256bad4, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1732539621122 2024-11-25T13:00:22,393 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 065eb16105c4aedcbfa7fb1ec2b14c86#B#compaction#562 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T13:00:22,393 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/662954a5cb9d4a18bc2e6345991a7fcc is 50, key is test_row_0/B:col10/1732539621136/Put/seqid=0 2024-11-25T13:00:22,394 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=065eb16105c4aedcbfa7fb1ec2b14c86] 2024-11-25T13:00:22,400 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411254bae50d75fcb446aa8a174c202ef843d_065eb16105c4aedcbfa7fb1ec2b14c86 store=[table=TestAcidGuarantees family=A region=065eb16105c4aedcbfa7fb1ec2b14c86] 2024-11-25T13:00:22,406 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411254bae50d75fcb446aa8a174c202ef843d_065eb16105c4aedcbfa7fb1ec2b14c86, store=[table=TestAcidGuarantees family=A region=065eb16105c4aedcbfa7fb1ec2b14c86] 2024-11-25T13:00:22,406 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411254bae50d75fcb446aa8a174c202ef843d_065eb16105c4aedcbfa7fb1ec2b14c86 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=065eb16105c4aedcbfa7fb1ec2b14c86] 2024-11-25T13:00:22,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742502_1678 (size=13425) 2024-11-25T13:00:22,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742503_1679 (size=4469) 2024-11-25T13:00:22,844 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/662954a5cb9d4a18bc2e6345991a7fcc as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/662954a5cb9d4a18bc2e6345991a7fcc 2024-11-25T13:00:22,849 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 065eb16105c4aedcbfa7fb1ec2b14c86/B of 065eb16105c4aedcbfa7fb1ec2b14c86 into 662954a5cb9d4a18bc2e6345991a7fcc(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T13:00:22,849 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 065eb16105c4aedcbfa7fb1ec2b14c86: 2024-11-25T13:00:22,849 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., storeName=065eb16105c4aedcbfa7fb1ec2b14c86/B, priority=13, startTime=1732539622383; duration=0sec 2024-11-25T13:00:22,849 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T13:00:22,849 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 065eb16105c4aedcbfa7fb1ec2b14c86:B 2024-11-25T13:00:22,849 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T13:00:22,850 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T13:00:22,850 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1541): 065eb16105c4aedcbfa7fb1ec2b14c86/C is initiating minor compaction (all files) 2024-11-25T13:00:22,850 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 065eb16105c4aedcbfa7fb1ec2b14c86/C in TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:22,850 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/557de74cc5f946e286f119fff14867a7, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/a908404ffd2049a594bc8e97847b5d3f, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/8fc3d1eae65e4dd98de707b35c98e96f] into tmpdir=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp, totalSize=37.0 K 2024-11-25T13:00:22,850 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 557de74cc5f946e286f119fff14867a7, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1732539617688 2024-11-25T13:00:22,850 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting a908404ffd2049a594bc8e97847b5d3f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1732539618930 2024-11-25T13:00:22,850 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] compactions.Compactor(225): Compacting 8fc3d1eae65e4dd98de707b35c98e96f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1732539621136 2024-11-25T13:00:22,853 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 065eb16105c4aedcbfa7fb1ec2b14c86#A#compaction#563 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T13:00:22,853 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/031876a032f74bdd89e2c0ae6dfeba42 is 175, key is test_row_0/A:col10/1732539621136/Put/seqid=0 2024-11-25T13:00:22,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742504_1680 (size=32379) 2024-11-25T13:00:22,864 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 065eb16105c4aedcbfa7fb1ec2b14c86#C#compaction#564 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T13:00:22,864 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/7565f9d2aa8547caa52b5b5bba6cfe6a is 50, key is test_row_0/C:col10/1732539621136/Put/seqid=0 2024-11-25T13:00:22,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742505_1681 (size=13391) 2024-11-25T13:00:23,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-25T13:00:23,210 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T13:00:23,266 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/031876a032f74bdd89e2c0ae6dfeba42 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/031876a032f74bdd89e2c0ae6dfeba42 2024-11-25T13:00:23,288 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 065eb16105c4aedcbfa7fb1ec2b14c86/A of 065eb16105c4aedcbfa7fb1ec2b14c86 into 031876a032f74bdd89e2c0ae6dfeba42(size=31.6 K), total size for store is 31.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T13:00:23,288 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 065eb16105c4aedcbfa7fb1ec2b14c86: 2024-11-25T13:00:23,288 INFO [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., storeName=065eb16105c4aedcbfa7fb1ec2b14c86/A, priority=13, startTime=1732539622383; duration=0sec 2024-11-25T13:00:23,288 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T13:00:23,288 DEBUG [RS:0;1ff8009e463e:35905-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 065eb16105c4aedcbfa7fb1ec2b14c86:A 2024-11-25T13:00:23,289 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/7565f9d2aa8547caa52b5b5bba6cfe6a as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/7565f9d2aa8547caa52b5b5bba6cfe6a 2024-11-25T13:00:23,304 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 065eb16105c4aedcbfa7fb1ec2b14c86/C of 065eb16105c4aedcbfa7fb1ec2b14c86 into 7565f9d2aa8547caa52b5b5bba6cfe6a(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T13:00:23,304 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 065eb16105c4aedcbfa7fb1ec2b14c86: 2024-11-25T13:00:23,304 INFO [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., storeName=065eb16105c4aedcbfa7fb1ec2b14c86/C, priority=13, startTime=1732539622383; duration=0sec 2024-11-25T13:00:23,304 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T13:00:23,304 DEBUG [RS:0;1ff8009e463e:35905-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 065eb16105c4aedcbfa7fb1ec2b14c86:C 2024-11-25T13:00:26,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35905 {}] regionserver.HRegion(8855): Flush requested on 065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:26,752 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T13:00:26,752 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 065eb16105c4aedcbfa7fb1ec2b14c86 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-25T13:00:26,753 INFO [Thread-2436 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2436. 2024-11-25T13:00:26,753 DEBUG [Thread-2436 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T13:00:26,753 DEBUG [Thread-2436 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T13:00:26,753 DEBUG [Thread-2436 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T13:00:26,753 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T13:00:26,777 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125576e416de4b94efcae8adf2e5e3ef8b7_065eb16105c4aedcbfa7fb1ec2b14c86 is 50, key is test_row_0/A:col10/1732539621780/Put/seqid=0 2024-11-25T13:00:26,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742506_1682 (size=12454) 2024-11-25T13:00:26,790 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:26,796 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125576e416de4b94efcae8adf2e5e3ef8b7_065eb16105c4aedcbfa7fb1ec2b14c86 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125576e416de4b94efcae8adf2e5e3ef8b7_065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:26,796 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/af4f9870a86547839b2be0ae117a9068, store: [table=TestAcidGuarantees family=A region=065eb16105c4aedcbfa7fb1ec2b14c86] 2024-11-25T13:00:26,797 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/af4f9870a86547839b2be0ae117a9068 is 175, key is test_row_0/A:col10/1732539621780/Put/seqid=0 2024-11-25T13:00:26,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742507_1683 (size=31255) 2024-11-25T13:00:26,811 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=479, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/af4f9870a86547839b2be0ae117a9068 2024-11-25T13:00:26,824 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/e5c07d398df84425b999dfeeac6a19d7 is 50, key is test_row_0/B:col10/1732539621780/Put/seqid=0 2024-11-25T13:00:26,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742508_1684 (size=12301) 2024-11-25T13:00:27,229 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/e5c07d398df84425b999dfeeac6a19d7 2024-11-25T13:00:27,234 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/f70deab5d61a4e97aa5a03f632628758 is 50, key is test_row_0/C:col10/1732539621780/Put/seqid=0 2024-11-25T13:00:27,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742509_1685 (size=12301) 2024-11-25T13:00:27,243 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/f70deab5d61a4e97aa5a03f632628758 2024-11-25T13:00:27,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/af4f9870a86547839b2be0ae117a9068 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/af4f9870a86547839b2be0ae117a9068 2024-11-25T13:00:27,250 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/af4f9870a86547839b2be0ae117a9068, entries=150, sequenceid=479, filesize=30.5 K 2024-11-25T13:00:27,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/e5c07d398df84425b999dfeeac6a19d7 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/e5c07d398df84425b999dfeeac6a19d7 2024-11-25T13:00:27,261 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/e5c07d398df84425b999dfeeac6a19d7, entries=150, sequenceid=479, filesize=12.0 K 2024-11-25T13:00:27,262 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/f70deab5d61a4e97aa5a03f632628758 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/f70deab5d61a4e97aa5a03f632628758 2024-11-25T13:00:27,264 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/f70deab5d61a4e97aa5a03f632628758, entries=150, sequenceid=479, filesize=12.0 K 2024-11-25T13:00:27,265 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=0 B/0 for 065eb16105c4aedcbfa7fb1ec2b14c86 in 513ms, sequenceid=479, compaction requested=false 2024-11-25T13:00:27,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 065eb16105c4aedcbfa7fb1ec2b14c86: 2024-11-25T13:00:27,356 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-25T13:00:29,912 INFO [Thread-2442 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2442. 2024-11-25T13:00:29,912 DEBUG [Thread-2442 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T13:00:29,912 DEBUG [Thread-2442 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T13:00:29,912 DEBUG [Thread-2442 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T13:00:29,913 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T13:00:29,913 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-25T13:00:29,913 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 100 2024-11-25T13:00:29,913 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 37 2024-11-25T13:00:29,913 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 107 2024-11-25T13:00:29,913 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 122 2024-11-25T13:00:29,913 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 6 2024-11-25T13:00:29,913 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-25T13:00:29,913 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2391 2024-11-25T13:00:29,913 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2238 2024-11-25T13:00:29,913 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2220 2024-11-25T13:00:29,913 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2356 2024-11-25T13:00:29,913 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2229 2024-11-25T13:00:29,913 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-25T13:00:29,913 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T13:00:29,913 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T13:00:29,913 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T13:00:29,913 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T13:00:29,914 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T13:00:29,915 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-25T13:00:29,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-25T13:00:29,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-25T13:00:29,920 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732539629919"}]},"ts":"1732539629919"} 2024-11-25T13:00:29,921 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-25T13:00:29,922 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-25T13:00:29,923 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-25T13:00:29,925 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=162, ppid=161, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=065eb16105c4aedcbfa7fb1ec2b14c86, UNASSIGN}] 2024-11-25T13:00:29,925 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=162, ppid=161, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=065eb16105c4aedcbfa7fb1ec2b14c86, UNASSIGN 2024-11-25T13:00:29,928 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=162 updating hbase:meta row=065eb16105c4aedcbfa7fb1ec2b14c86, regionState=CLOSING, regionLocation=1ff8009e463e,35905,1732539449246 2024-11-25T13:00:29,930 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=162, ppid=161, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=065eb16105c4aedcbfa7fb1ec2b14c86, UNASSIGN because future has completed 2024-11-25T13:00:29,930 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-25T13:00:29,930 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; CloseRegionProcedure 065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246}] 2024-11-25T13:00:30,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-25T13:00:30,083 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] handler.UnassignRegionHandler(122): Close 065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:30,084 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-25T13:00:30,084 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1722): Closing 065eb16105c4aedcbfa7fb1ec2b14c86, disabling compactions & flushes 2024-11-25T13:00:30,084 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:30,084 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:30,084 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. after waiting 0 ms 2024-11-25T13:00:30,084 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:30,084 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(2902): Flushing 065eb16105c4aedcbfa7fb1ec2b14c86 3/3 column families, dataSize=6.71 KB heapSize=18.33 KB 2024-11-25T13:00:30,088 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112574b67f83f5ce4c6db0d11dde5c628ed7_065eb16105c4aedcbfa7fb1ec2b14c86 is 50, key is test_row_2/A:col10/1732539629911/Put/seqid=0 2024-11-25T13:00:30,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742510_1686 (size=7374) 2024-11-25T13:00:30,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-25T13:00:30,492 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T13:00:30,501 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112574b67f83f5ce4c6db0d11dde5c628ed7_065eb16105c4aedcbfa7fb1ec2b14c86 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112574b67f83f5ce4c6db0d11dde5c628ed7_065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:30,502 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/a9871786f8144007bfa9bc8bc68875da, store: [table=TestAcidGuarantees family=A region=065eb16105c4aedcbfa7fb1ec2b14c86] 2024-11-25T13:00:30,503 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/a9871786f8144007bfa9bc8bc68875da is 175, key is test_row_2/A:col10/1732539629911/Put/seqid=0 2024-11-25T13:00:30,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742511_1687 (size=13865) 2024-11-25T13:00:30,515 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=483, memsize=2.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/a9871786f8144007bfa9bc8bc68875da 2024-11-25T13:00:30,529 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/cbb9f8a665eb4896a428622c4c6f2063 is 50, key is test_row_2/B:col10/1732539629911/Put/seqid=0 2024-11-25T13:00:30,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-25T13:00:30,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742512_1688 (size=7415) 2024-11-25T13:00:30,554 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=483 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/cbb9f8a665eb4896a428622c4c6f2063 2024-11-25T13:00:30,561 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/3fee0dba85ef477887e974a0992f2852 is 50, key is test_row_2/C:col10/1732539629911/Put/seqid=0 2024-11-25T13:00:30,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742513_1689 (size=7415) 2024-11-25T13:00:30,570 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=483 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/3fee0dba85ef477887e974a0992f2852 2024-11-25T13:00:30,587 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/A/a9871786f8144007bfa9bc8bc68875da as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/a9871786f8144007bfa9bc8bc68875da 2024-11-25T13:00:30,593 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/a9871786f8144007bfa9bc8bc68875da, entries=50, sequenceid=483, filesize=13.5 K 2024-11-25T13:00:30,594 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/B/cbb9f8a665eb4896a428622c4c6f2063 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/cbb9f8a665eb4896a428622c4c6f2063 2024-11-25T13:00:30,602 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/cbb9f8a665eb4896a428622c4c6f2063, entries=50, sequenceid=483, filesize=7.2 K 2024-11-25T13:00:30,603 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/.tmp/C/3fee0dba85ef477887e974a0992f2852 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/3fee0dba85ef477887e974a0992f2852 2024-11-25T13:00:30,607 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/3fee0dba85ef477887e974a0992f2852, entries=50, sequenceid=483, filesize=7.2 K 2024-11-25T13:00:30,608 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~6.71 KB/6870, heapSize ~18.28 KB/18720, currentSize=0 B/0 for 065eb16105c4aedcbfa7fb1ec2b14c86 in 524ms, sequenceid=483, compaction requested=true 2024-11-25T13:00:30,609 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/c8fa4d5e1cff4ce7af133036fcaa5c96, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/628f46655ebf45778131f9f8b76caf88, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/06b361f16ce84fd2a4eaff947ed16b69, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/71f8c18f61bd4fab838ff17930b4a166, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/f4c8294f23094e2688c5f02256bc3289, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/d16f399185e640bcab04b23f289866e5, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/5fbd81d77b744594a5c4bb38d52feb8b, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/917b97f9ed504fc880b8f6529317ffb8, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/dc9d76ac80a64714a17964d357ff8e4d, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/6bde8b420bb949e7b3b1278988b2163f, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/2ddca797c1e94562a5f17432498e94c8, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/4cea6c322769443b946345458855893a, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/f5af782a93704dc49ccd218e18bbb3b2, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/7049665fd88b448c9f277d56be1af121, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/d79633c7018742b5afa9c5f8d1352e13, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/ef2e4fc7e2aa44e2bf928f33bcefb473, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/edf64eadc2f64469b146d56748034ae9, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/f5e3d78c0ca0497bb52bafe1322a4985, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/ec95b895116a443d881d96facd416497, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/8b9b8b5a969e40e8a07bf94fcd1b4135, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/fb97de48734c4b22b3c5ad4dd1c2e73b, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/e417e9cf7d4d48b99c5359ebf406c6b6, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/0af3647a9f194c8d8acabadf61612cd6, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/8d63686e01de4235a4e6d1acacb29e14, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/e11998640e9d4797a380116020fbd101, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/4572d0dfd4b54300b33afdabe7a22eac, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/0637c5ff8ea047438721477de61f580b, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/5b5171ebe1d8486a908063d9e37d207b, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/93aa332746564b6c8c96da12ab91d882, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/7df79d6fa00a40d2adfe9a7eea4d1ca2, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/00c0ad8b01dc43649b7da48b28deef08, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/0f6bb67859bb4056a09638496ca2b0c1, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/3d14e3987e7b4026b59e6f1f2256bad4] to archive 2024-11-25T13:00:30,610 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T13:00:30,613 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/c8fa4d5e1cff4ce7af133036fcaa5c96 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/c8fa4d5e1cff4ce7af133036fcaa5c96 2024-11-25T13:00:30,615 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/628f46655ebf45778131f9f8b76caf88 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/628f46655ebf45778131f9f8b76caf88 2024-11-25T13:00:30,629 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/06b361f16ce84fd2a4eaff947ed16b69 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/06b361f16ce84fd2a4eaff947ed16b69 2024-11-25T13:00:30,639 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/71f8c18f61bd4fab838ff17930b4a166 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/71f8c18f61bd4fab838ff17930b4a166 2024-11-25T13:00:30,641 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/f4c8294f23094e2688c5f02256bc3289 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/f4c8294f23094e2688c5f02256bc3289 2024-11-25T13:00:30,642 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/d16f399185e640bcab04b23f289866e5 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/d16f399185e640bcab04b23f289866e5 2024-11-25T13:00:30,646 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/5fbd81d77b744594a5c4bb38d52feb8b to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/5fbd81d77b744594a5c4bb38d52feb8b 2024-11-25T13:00:30,651 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/917b97f9ed504fc880b8f6529317ffb8 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/917b97f9ed504fc880b8f6529317ffb8 2024-11-25T13:00:30,658 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/dc9d76ac80a64714a17964d357ff8e4d to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/dc9d76ac80a64714a17964d357ff8e4d 2024-11-25T13:00:30,659 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/6bde8b420bb949e7b3b1278988b2163f to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/6bde8b420bb949e7b3b1278988b2163f 2024-11-25T13:00:30,661 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/2ddca797c1e94562a5f17432498e94c8 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/2ddca797c1e94562a5f17432498e94c8 2024-11-25T13:00:30,663 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/4cea6c322769443b946345458855893a to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/4cea6c322769443b946345458855893a 2024-11-25T13:00:30,664 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/f5af782a93704dc49ccd218e18bbb3b2 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/f5af782a93704dc49ccd218e18bbb3b2 2024-11-25T13:00:30,666 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/7049665fd88b448c9f277d56be1af121 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/7049665fd88b448c9f277d56be1af121 2024-11-25T13:00:30,667 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/d79633c7018742b5afa9c5f8d1352e13 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/d79633c7018742b5afa9c5f8d1352e13 2024-11-25T13:00:30,670 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/ef2e4fc7e2aa44e2bf928f33bcefb473 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/ef2e4fc7e2aa44e2bf928f33bcefb473 2024-11-25T13:00:30,675 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/edf64eadc2f64469b146d56748034ae9 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/edf64eadc2f64469b146d56748034ae9 2024-11-25T13:00:30,681 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/f5e3d78c0ca0497bb52bafe1322a4985 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/f5e3d78c0ca0497bb52bafe1322a4985 2024-11-25T13:00:30,682 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/ec95b895116a443d881d96facd416497 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/ec95b895116a443d881d96facd416497 2024-11-25T13:00:30,684 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/8b9b8b5a969e40e8a07bf94fcd1b4135 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/8b9b8b5a969e40e8a07bf94fcd1b4135 2024-11-25T13:00:30,689 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/fb97de48734c4b22b3c5ad4dd1c2e73b to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/fb97de48734c4b22b3c5ad4dd1c2e73b 2024-11-25T13:00:30,692 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/e417e9cf7d4d48b99c5359ebf406c6b6 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/e417e9cf7d4d48b99c5359ebf406c6b6 2024-11-25T13:00:30,696 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/0af3647a9f194c8d8acabadf61612cd6 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/0af3647a9f194c8d8acabadf61612cd6 2024-11-25T13:00:30,697 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/8d63686e01de4235a4e6d1acacb29e14 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/8d63686e01de4235a4e6d1acacb29e14 2024-11-25T13:00:30,698 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/e11998640e9d4797a380116020fbd101 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/e11998640e9d4797a380116020fbd101 2024-11-25T13:00:30,700 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/4572d0dfd4b54300b33afdabe7a22eac to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/4572d0dfd4b54300b33afdabe7a22eac 2024-11-25T13:00:30,701 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/0637c5ff8ea047438721477de61f580b to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/0637c5ff8ea047438721477de61f580b 2024-11-25T13:00:30,702 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/5b5171ebe1d8486a908063d9e37d207b to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/5b5171ebe1d8486a908063d9e37d207b 2024-11-25T13:00:30,709 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/93aa332746564b6c8c96da12ab91d882 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/93aa332746564b6c8c96da12ab91d882 2024-11-25T13:00:30,710 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/7df79d6fa00a40d2adfe9a7eea4d1ca2 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/7df79d6fa00a40d2adfe9a7eea4d1ca2 2024-11-25T13:00:30,716 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/00c0ad8b01dc43649b7da48b28deef08 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/00c0ad8b01dc43649b7da48b28deef08 2024-11-25T13:00:30,718 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/0f6bb67859bb4056a09638496ca2b0c1 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/0f6bb67859bb4056a09638496ca2b0c1 2024-11-25T13:00:30,721 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/3d14e3987e7b4026b59e6f1f2256bad4 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/3d14e3987e7b4026b59e6f1f2256bad4 2024-11-25T13:00:30,731 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/a85fb91fd49b47f3be87d55cc4ce7d09, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/b4882718154f4426abb1ed8dbe384cd9, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/0a8fd354251e449ea976e954aef6c896, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/b186d3dc5e1048e4a094684416935727, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/f202eddfeafb4fddbfc6c93379e91569, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/bf5387e09c534cea819c5380096bb271, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/e9f7da0583744882a2853c1fe7c9502d, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/28af56c923014440b0e0f9781864d8aa, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/b9f126dfaa1842f3aea1d44697469793, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/65d601588e17457cb335f172d7d1832f, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/7268c14c7213414182cbcc59bf30a206, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/8a80df7c7c384bfe9d30d4c7783b0337, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/80f834c5e1134ceeb1fb5b4a9692e5a0, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/c57a6e0c74b440f39069e5c058e97fe2, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/689521bfe2ad4d15b86b33f2a53b618d, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/2266e99c8d7147e8a93ed9741665d03b, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/a83fe6beadb6421f988d45c664b90c22, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/157862308be74a0aba128e555e4bba2f, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/f63048269c274fbeb02a501b7d42e592, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/1b9c70ff633840049de5c332838876bf, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/e797a4811a274412bf15f54ce9a3d1ee, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/d181f40310ca463284c6d715fdb69413, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/d8c71ab023c440c7bde170e89573dbbb, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/d173f591b4b44eb2bf29021c430a74fe, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/b50ff63861e5482e8ad93e7efe1a4b9e, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/3adc37aad8ee4b4cb13a637c2684de06, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/c5202bf19a014d1280d357593232e9b1, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/3f13d2e1d267489d900c464ca8f9dd35, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/584e3c4a1a914930bf833950fb8f3fa8, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/2a5174cbfee142eb898f2de1a3d11284, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/d3319f77a810494faf3247091d1acec1, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/09aed9bc5a914e1897ef4ced07e6a3f4, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/a62e40c9be9a48a58bb5ff8e3a7b7448] to archive 2024-11-25T13:00:30,732 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T13:00:30,736 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/a85fb91fd49b47f3be87d55cc4ce7d09 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/a85fb91fd49b47f3be87d55cc4ce7d09 2024-11-25T13:00:30,737 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/b4882718154f4426abb1ed8dbe384cd9 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/b4882718154f4426abb1ed8dbe384cd9 2024-11-25T13:00:30,738 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/0a8fd354251e449ea976e954aef6c896 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/0a8fd354251e449ea976e954aef6c896 2024-11-25T13:00:30,739 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/b186d3dc5e1048e4a094684416935727 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/b186d3dc5e1048e4a094684416935727 2024-11-25T13:00:30,739 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/f202eddfeafb4fddbfc6c93379e91569 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/f202eddfeafb4fddbfc6c93379e91569 2024-11-25T13:00:30,740 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/bf5387e09c534cea819c5380096bb271 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/bf5387e09c534cea819c5380096bb271 2024-11-25T13:00:30,741 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/e9f7da0583744882a2853c1fe7c9502d to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/e9f7da0583744882a2853c1fe7c9502d 2024-11-25T13:00:30,742 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/28af56c923014440b0e0f9781864d8aa to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/28af56c923014440b0e0f9781864d8aa 2024-11-25T13:00:30,742 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/b9f126dfaa1842f3aea1d44697469793 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/b9f126dfaa1842f3aea1d44697469793 2024-11-25T13:00:30,743 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/65d601588e17457cb335f172d7d1832f to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/65d601588e17457cb335f172d7d1832f 2024-11-25T13:00:30,744 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/7268c14c7213414182cbcc59bf30a206 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/7268c14c7213414182cbcc59bf30a206 2024-11-25T13:00:30,745 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/8a80df7c7c384bfe9d30d4c7783b0337 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/8a80df7c7c384bfe9d30d4c7783b0337 2024-11-25T13:00:30,746 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/80f834c5e1134ceeb1fb5b4a9692e5a0 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/80f834c5e1134ceeb1fb5b4a9692e5a0 2024-11-25T13:00:30,747 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/c57a6e0c74b440f39069e5c058e97fe2 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/c57a6e0c74b440f39069e5c058e97fe2 2024-11-25T13:00:30,748 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/689521bfe2ad4d15b86b33f2a53b618d to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/689521bfe2ad4d15b86b33f2a53b618d 2024-11-25T13:00:30,749 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/2266e99c8d7147e8a93ed9741665d03b to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/2266e99c8d7147e8a93ed9741665d03b 2024-11-25T13:00:30,750 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/a83fe6beadb6421f988d45c664b90c22 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/a83fe6beadb6421f988d45c664b90c22 2024-11-25T13:00:30,751 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/157862308be74a0aba128e555e4bba2f to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/157862308be74a0aba128e555e4bba2f 2024-11-25T13:00:30,751 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/f63048269c274fbeb02a501b7d42e592 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/f63048269c274fbeb02a501b7d42e592 2024-11-25T13:00:30,752 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/1b9c70ff633840049de5c332838876bf to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/1b9c70ff633840049de5c332838876bf 2024-11-25T13:00:30,753 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/e797a4811a274412bf15f54ce9a3d1ee to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/e797a4811a274412bf15f54ce9a3d1ee 2024-11-25T13:00:30,754 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/d181f40310ca463284c6d715fdb69413 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/d181f40310ca463284c6d715fdb69413 2024-11-25T13:00:30,755 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/d8c71ab023c440c7bde170e89573dbbb to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/d8c71ab023c440c7bde170e89573dbbb 2024-11-25T13:00:30,755 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/d173f591b4b44eb2bf29021c430a74fe to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/d173f591b4b44eb2bf29021c430a74fe 2024-11-25T13:00:30,756 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/b50ff63861e5482e8ad93e7efe1a4b9e to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/b50ff63861e5482e8ad93e7efe1a4b9e 2024-11-25T13:00:30,757 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/3adc37aad8ee4b4cb13a637c2684de06 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/3adc37aad8ee4b4cb13a637c2684de06 2024-11-25T13:00:30,758 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/c5202bf19a014d1280d357593232e9b1 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/c5202bf19a014d1280d357593232e9b1 2024-11-25T13:00:30,759 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/3f13d2e1d267489d900c464ca8f9dd35 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/3f13d2e1d267489d900c464ca8f9dd35 2024-11-25T13:00:30,760 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/584e3c4a1a914930bf833950fb8f3fa8 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/584e3c4a1a914930bf833950fb8f3fa8 2024-11-25T13:00:30,760 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/2a5174cbfee142eb898f2de1a3d11284 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/2a5174cbfee142eb898f2de1a3d11284 2024-11-25T13:00:30,762 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/d3319f77a810494faf3247091d1acec1 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/d3319f77a810494faf3247091d1acec1 2024-11-25T13:00:30,763 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/09aed9bc5a914e1897ef4ced07e6a3f4 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/09aed9bc5a914e1897ef4ced07e6a3f4 2024-11-25T13:00:30,765 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/a62e40c9be9a48a58bb5ff8e3a7b7448 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/a62e40c9be9a48a58bb5ff8e3a7b7448 2024-11-25T13:00:30,767 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/9ed428103bca4174b11c33b713045b89, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/f8ff43fb7780480e8508e5edf99d76d2, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/b09f679fdb0049d79c24ee9928b7ebf5, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/897730c0e49945dda8098bafb3d628d8, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/2a60d1c95d444d7caddcd5f7b7358050, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/a9a1e5503d374888ab55d8b96206586d, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/79cc8d29a6c54f48889a051eeaafe0b0, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/6257f9db72f743d8ae7474c663c0b0c7, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/6885c0ca502d47d995bb0d8948f8b9f5, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/ac11773566104d58ac71b3384593cc57, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/e54c84cf8c374916b5d6622fe1a18c68, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/ca4eb6741a4148fa84cb6ef250754549, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/f912f897630a4d068d4b9a9bc5eea475, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/6b5a43ae991c48a3a45d4a6d78d80ecf, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/b12e02a3f6564b1a9b47ca43ae2626b2, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/394b57743e5347f69e9a9824f1da4c26, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/646aae351bbd4c54b84aa6a6e04db69e, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/102712bae900483284798a98e5133a37, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/b66cc7274d494fd2b929d6752b58d249, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/ba2f61c5cc914c73ada04671ae67aab3, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/33eeac8f284f49d88cda8d3bae9a7fd2, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/6e1cbfee3adf4626b9e94903ccfb0754, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/e43622462eab46a886ac24aea6041446, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/7df74bdf219c4e16b1ebd36383f64716, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/38b2934fc17a4928a610ee204a6f5106, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/a6123c68353a4d0099f9619344da39cd, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/a1ae7e8c93774652a12525feba255ac2, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/aa6b3b9e20f64a5797cdfbe9f2b1cc2b, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/557de74cc5f946e286f119fff14867a7, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/3a24aab432904317826b53dbdf6f4c39, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/a908404ffd2049a594bc8e97847b5d3f, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/8fc3d1eae65e4dd98de707b35c98e96f] to archive 2024-11-25T13:00:30,771 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T13:00:30,773 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/9ed428103bca4174b11c33b713045b89 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/9ed428103bca4174b11c33b713045b89 2024-11-25T13:00:30,774 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/f8ff43fb7780480e8508e5edf99d76d2 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/f8ff43fb7780480e8508e5edf99d76d2 2024-11-25T13:00:30,775 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/b09f679fdb0049d79c24ee9928b7ebf5 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/b09f679fdb0049d79c24ee9928b7ebf5 2024-11-25T13:00:30,776 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/897730c0e49945dda8098bafb3d628d8 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/897730c0e49945dda8098bafb3d628d8 2024-11-25T13:00:30,777 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/2a60d1c95d444d7caddcd5f7b7358050 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/2a60d1c95d444d7caddcd5f7b7358050 2024-11-25T13:00:30,778 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/a9a1e5503d374888ab55d8b96206586d to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/a9a1e5503d374888ab55d8b96206586d 2024-11-25T13:00:30,779 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/79cc8d29a6c54f48889a051eeaafe0b0 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/79cc8d29a6c54f48889a051eeaafe0b0 2024-11-25T13:00:30,780 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/6257f9db72f743d8ae7474c663c0b0c7 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/6257f9db72f743d8ae7474c663c0b0c7 2024-11-25T13:00:30,781 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/6885c0ca502d47d995bb0d8948f8b9f5 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/6885c0ca502d47d995bb0d8948f8b9f5 2024-11-25T13:00:30,782 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/ac11773566104d58ac71b3384593cc57 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/ac11773566104d58ac71b3384593cc57 2024-11-25T13:00:30,783 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/e54c84cf8c374916b5d6622fe1a18c68 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/e54c84cf8c374916b5d6622fe1a18c68 2024-11-25T13:00:30,784 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/ca4eb6741a4148fa84cb6ef250754549 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/ca4eb6741a4148fa84cb6ef250754549 2024-11-25T13:00:30,785 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/f912f897630a4d068d4b9a9bc5eea475 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/f912f897630a4d068d4b9a9bc5eea475 2024-11-25T13:00:30,785 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/6b5a43ae991c48a3a45d4a6d78d80ecf to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/6b5a43ae991c48a3a45d4a6d78d80ecf 2024-11-25T13:00:30,788 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/b12e02a3f6564b1a9b47ca43ae2626b2 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/b12e02a3f6564b1a9b47ca43ae2626b2 2024-11-25T13:00:30,789 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/394b57743e5347f69e9a9824f1da4c26 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/394b57743e5347f69e9a9824f1da4c26 2024-11-25T13:00:30,790 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/646aae351bbd4c54b84aa6a6e04db69e to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/646aae351bbd4c54b84aa6a6e04db69e 2024-11-25T13:00:30,791 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/102712bae900483284798a98e5133a37 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/102712bae900483284798a98e5133a37 2024-11-25T13:00:30,792 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/b66cc7274d494fd2b929d6752b58d249 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/b66cc7274d494fd2b929d6752b58d249 2024-11-25T13:00:30,793 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/ba2f61c5cc914c73ada04671ae67aab3 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/ba2f61c5cc914c73ada04671ae67aab3 2024-11-25T13:00:30,793 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/33eeac8f284f49d88cda8d3bae9a7fd2 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/33eeac8f284f49d88cda8d3bae9a7fd2 2024-11-25T13:00:30,794 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/6e1cbfee3adf4626b9e94903ccfb0754 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/6e1cbfee3adf4626b9e94903ccfb0754 2024-11-25T13:00:30,795 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/e43622462eab46a886ac24aea6041446 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/e43622462eab46a886ac24aea6041446 2024-11-25T13:00:30,796 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/7df74bdf219c4e16b1ebd36383f64716 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/7df74bdf219c4e16b1ebd36383f64716 2024-11-25T13:00:30,796 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/38b2934fc17a4928a610ee204a6f5106 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/38b2934fc17a4928a610ee204a6f5106 2024-11-25T13:00:30,797 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/a6123c68353a4d0099f9619344da39cd to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/a6123c68353a4d0099f9619344da39cd 2024-11-25T13:00:30,798 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/a1ae7e8c93774652a12525feba255ac2 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/a1ae7e8c93774652a12525feba255ac2 2024-11-25T13:00:30,798 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/aa6b3b9e20f64a5797cdfbe9f2b1cc2b to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/aa6b3b9e20f64a5797cdfbe9f2b1cc2b 2024-11-25T13:00:30,799 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/557de74cc5f946e286f119fff14867a7 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/557de74cc5f946e286f119fff14867a7 2024-11-25T13:00:30,800 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/3a24aab432904317826b53dbdf6f4c39 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/3a24aab432904317826b53dbdf6f4c39 2024-11-25T13:00:30,801 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/a908404ffd2049a594bc8e97847b5d3f to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/a908404ffd2049a594bc8e97847b5d3f 2024-11-25T13:00:30,801 DEBUG [StoreCloser-TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/8fc3d1eae65e4dd98de707b35c98e96f to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/8fc3d1eae65e4dd98de707b35c98e96f 2024-11-25T13:00:30,811 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/recovered.edits/486.seqid, newMaxSeqId=486, maxSeqId=4 2024-11-25T13:00:30,812 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. 2024-11-25T13:00:30,812 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1676): Region close journal for 065eb16105c4aedcbfa7fb1ec2b14c86: Waiting for close lock at 1732539630084Running coprocessor pre-close hooks at 1732539630084Disabling compacts and flushes for region at 1732539630084Disabling writes for close at 1732539630084Obtaining lock to block concurrent updates at 1732539630084Preparing flush snapshotting stores in 065eb16105c4aedcbfa7fb1ec2b14c86 at 1732539630084Finished memstore snapshotting TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86., syncing WAL and waiting on mvcc, flushsize=dataSize=6870, getHeapSize=18720, getOffHeapSize=0, getCellsCount=150 at 1732539630084Flushing stores of TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86. at 1732539630085 (+1 ms)Flushing 065eb16105c4aedcbfa7fb1ec2b14c86/A: creating writer at 1732539630085Flushing mob file 065eb16105c4aedcbfa7fb1ec2b14c86/A: appending metadata at 1732539630088 (+3 ms)Flushing mob file 065eb16105c4aedcbfa7fb1ec2b14c86/A: closing flushed file at 1732539630088Flushing 065eb16105c4aedcbfa7fb1ec2b14c86/A: appending metadata at 1732539630502 (+414 ms)Flushing 065eb16105c4aedcbfa7fb1ec2b14c86/A: closing flushed file at 1732539630502Flushing 065eb16105c4aedcbfa7fb1ec2b14c86/B: creating writer at 1732539630524 (+22 ms)Flushing 065eb16105c4aedcbfa7fb1ec2b14c86/B: appending metadata at 1732539630528 (+4 ms)Flushing 065eb16105c4aedcbfa7fb1ec2b14c86/B: closing flushed file at 1732539630528Flushing 065eb16105c4aedcbfa7fb1ec2b14c86/C: creating writer at 1732539630558 (+30 ms)Flushing 065eb16105c4aedcbfa7fb1ec2b14c86/C: appending metadata at 1732539630560 (+2 ms)Flushing 065eb16105c4aedcbfa7fb1ec2b14c86/C: closing flushed file at 1732539630560Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@26d00433: reopening flushed file at 1732539630575 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@77a85a62: reopening flushed file at 1732539630593 (+18 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7c49dd20: reopening flushed file at 1732539630602 (+9 ms)Finished flush of dataSize ~6.71 KB/6870, heapSize ~18.28 KB/18720, currentSize=0 B/0 for 065eb16105c4aedcbfa7fb1ec2b14c86 in 524ms, sequenceid=483, compaction requested=true at 1732539630608 (+6 ms)Writing region close event to WAL at 1732539630803 (+195 ms)Running coprocessor post-close hooks at 1732539630812 (+9 ms)Closed at 1732539630812 2024-11-25T13:00:30,814 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] handler.UnassignRegionHandler(157): Closed 065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:30,815 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=162 updating hbase:meta row=065eb16105c4aedcbfa7fb1ec2b14c86, regionState=CLOSED 2024-11-25T13:00:30,818 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=163, ppid=162, state=RUNNABLE, hasLock=false; CloseRegionProcedure 065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 because future has completed 2024-11-25T13:00:30,821 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-25T13:00:30,821 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; CloseRegionProcedure 065eb16105c4aedcbfa7fb1ec2b14c86, server=1ff8009e463e,35905,1732539449246 in 889 msec 2024-11-25T13:00:30,823 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=162, resume processing ppid=161 2024-11-25T13:00:30,823 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, ppid=161, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=065eb16105c4aedcbfa7fb1ec2b14c86, UNASSIGN in 897 msec 2024-11-25T13:00:30,826 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-25T13:00:30,826 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 901 msec 2024-11-25T13:00:30,828 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732539630828"}]},"ts":"1732539630828"} 2024-11-25T13:00:30,837 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-25T13:00:30,837 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-25T13:00:30,839 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 923 msec 2024-11-25T13:00:31,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-25T13:00:31,067 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-25T13:00:31,067 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-25T13:00:31,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T13:00:31,069 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=164, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T13:00:31,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-25T13:00:31,072 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=164, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T13:00:31,075 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:31,089 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A, FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B, FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C, FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/recovered.edits] 2024-11-25T13:00:31,096 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/031876a032f74bdd89e2c0ae6dfeba42 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/031876a032f74bdd89e2c0ae6dfeba42 2024-11-25T13:00:31,108 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/a9871786f8144007bfa9bc8bc68875da to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/a9871786f8144007bfa9bc8bc68875da 2024-11-25T13:00:31,110 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/af4f9870a86547839b2be0ae117a9068 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/A/af4f9870a86547839b2be0ae117a9068 2024-11-25T13:00:31,136 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/662954a5cb9d4a18bc2e6345991a7fcc to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/662954a5cb9d4a18bc2e6345991a7fcc 2024-11-25T13:00:31,137 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/cbb9f8a665eb4896a428622c4c6f2063 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/cbb9f8a665eb4896a428622c4c6f2063 2024-11-25T13:00:31,138 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/e5c07d398df84425b999dfeeac6a19d7 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/B/e5c07d398df84425b999dfeeac6a19d7 2024-11-25T13:00:31,140 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/3fee0dba85ef477887e974a0992f2852 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/3fee0dba85ef477887e974a0992f2852 2024-11-25T13:00:31,141 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/7565f9d2aa8547caa52b5b5bba6cfe6a to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/7565f9d2aa8547caa52b5b5bba6cfe6a 2024-11-25T13:00:31,152 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/f70deab5d61a4e97aa5a03f632628758 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/C/f70deab5d61a4e97aa5a03f632628758 2024-11-25T13:00:31,154 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/recovered.edits/486.seqid to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86/recovered.edits/486.seqid 2024-11-25T13:00:31,155 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/default/TestAcidGuarantees/065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:31,155 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-25T13:00:31,156 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-25T13:00:31,156 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-25T13:00:31,159 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411250a1f611364c24c81adeed21b716e1d66_065eb16105c4aedcbfa7fb1ec2b14c86 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411250a1f611364c24c81adeed21b716e1d66_065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:31,161 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411250b186797ba3f4fcaacd663c47d10035c_065eb16105c4aedcbfa7fb1ec2b14c86 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411250b186797ba3f4fcaacd663c47d10035c_065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:31,162 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411251c935c2fea814d1db1a0a867d1a7b1e5_065eb16105c4aedcbfa7fb1ec2b14c86 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411251c935c2fea814d1db1a0a867d1a7b1e5_065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:31,163 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112521ff85d72ba14b2b9b87b89e012941f6_065eb16105c4aedcbfa7fb1ec2b14c86 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112521ff85d72ba14b2b9b87b89e012941f6_065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:31,164 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411252ac9b8682fe04ca49f58eb13ce359fd4_065eb16105c4aedcbfa7fb1ec2b14c86 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411252ac9b8682fe04ca49f58eb13ce359fd4_065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:31,166 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411253a0f50a6a0cf48b6adafaafd9d2c9579_065eb16105c4aedcbfa7fb1ec2b14c86 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411253a0f50a6a0cf48b6adafaafd9d2c9579_065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:31,167 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125576e416de4b94efcae8adf2e5e3ef8b7_065eb16105c4aedcbfa7fb1ec2b14c86 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125576e416de4b94efcae8adf2e5e3ef8b7_065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:31,168 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411255c98f39e253f4fcf8836e76b7d84340e_065eb16105c4aedcbfa7fb1ec2b14c86 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411255c98f39e253f4fcf8836e76b7d84340e_065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:31,169 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112562ed6d11882543f8b4200341d5e18879_065eb16105c4aedcbfa7fb1ec2b14c86 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112562ed6d11882543f8b4200341d5e18879_065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:31,170 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411256f9fc12468ab40aabc6f774897674d33_065eb16105c4aedcbfa7fb1ec2b14c86 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411256f9fc12468ab40aabc6f774897674d33_065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:31,171 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125732a9c8ca7594db99348f8a995a70382_065eb16105c4aedcbfa7fb1ec2b14c86 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125732a9c8ca7594db99348f8a995a70382_065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:31,172 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112574b67f83f5ce4c6db0d11dde5c628ed7_065eb16105c4aedcbfa7fb1ec2b14c86 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112574b67f83f5ce4c6db0d11dde5c628ed7_065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:31,173 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411258250882379e14b0da4668b1ba45330bd_065eb16105c4aedcbfa7fb1ec2b14c86 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411258250882379e14b0da4668b1ba45330bd_065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:31,174 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125980783caecf74820af244ee33cd10574_065eb16105c4aedcbfa7fb1ec2b14c86 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125980783caecf74820af244ee33cd10574_065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:31,175 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411259a9472b94dcb467e99fae9928c23e4d0_065eb16105c4aedcbfa7fb1ec2b14c86 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411259a9472b94dcb467e99fae9928c23e4d0_065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:31,176 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125a1f81cd22a0e48468bf67e9b5be3f06e_065eb16105c4aedcbfa7fb1ec2b14c86 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125a1f81cd22a0e48468bf67e9b5be3f06e_065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:31,177 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125a3940c8519b34aca99a3b380c7533869_065eb16105c4aedcbfa7fb1ec2b14c86 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125a3940c8519b34aca99a3b380c7533869_065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:31,178 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125a87d6582de754e589b481d1f2e2acc81_065eb16105c4aedcbfa7fb1ec2b14c86 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125a87d6582de754e589b481d1f2e2acc81_065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:31,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-25T13:00:31,181 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125aeb01705bb1c4151b7f0e0f3afc17c34_065eb16105c4aedcbfa7fb1ec2b14c86 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125aeb01705bb1c4151b7f0e0f3afc17c34_065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:31,182 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125cc58a260b5504369a0b103c79fda0b99_065eb16105c4aedcbfa7fb1ec2b14c86 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125cc58a260b5504369a0b103c79fda0b99_065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:31,183 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125cef94469a5754d7b8d6efd9010cdeb47_065eb16105c4aedcbfa7fb1ec2b14c86 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125cef94469a5754d7b8d6efd9010cdeb47_065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:31,187 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125e59dc0d1c1594bbfae08f02280d14d62_065eb16105c4aedcbfa7fb1ec2b14c86 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125e59dc0d1c1594bbfae08f02280d14d62_065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:31,203 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125e8996a5b2a3343bfbcb7dd493fb72075_065eb16105c4aedcbfa7fb1ec2b14c86 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125e8996a5b2a3343bfbcb7dd493fb72075_065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:31,220 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125ec694a9bbd2f403ba6a1d52d45bc9026_065eb16105c4aedcbfa7fb1ec2b14c86 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125ec694a9bbd2f403ba6a1d52d45bc9026_065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:31,242 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125ee23a9e77ae0449a897f1c4554d432a6_065eb16105c4aedcbfa7fb1ec2b14c86 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125ee23a9e77ae0449a897f1c4554d432a6_065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:31,262 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125f3421c23089148c69d27092ac8490b81_065eb16105c4aedcbfa7fb1ec2b14c86 to hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125f3421c23089148c69d27092ac8490b81_065eb16105c4aedcbfa7fb1ec2b14c86 2024-11-25T13:00:31,264 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-25T13:00:31,272 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=164, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T13:00:31,276 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-25T13:00:31,278 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-25T13:00:31,279 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=164, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T13:00:31,279 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-25T13:00:31,283 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732539631280"}]},"ts":"9223372036854775807"} 2024-11-25T13:00:31,286 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-25T13:00:31,286 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 065eb16105c4aedcbfa7fb1ec2b14c86, NAME => 'TestAcidGuarantees,,1732539599290.065eb16105c4aedcbfa7fb1ec2b14c86.', STARTKEY => '', ENDKEY => ''}] 2024-11-25T13:00:31,286 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-25T13:00:31,286 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732539631286"}]},"ts":"9223372036854775807"} 2024-11-25T13:00:31,289 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-25T13:00:31,289 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=164, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T13:00:31,291 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 223 msec 2024-11-25T13:00:31,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-25T13:00:31,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-25T13:00:31,393 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-25T13:00:31,399 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=237 (was 240), OpenFileDescriptor=449 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1017 (was 1045), ProcessCount=11 (was 11), AvailableMemoryMB=2404 (was 674) - AvailableMemoryMB LEAK? - 2024-11-25T13:00:31,399 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-25T13:00:31,399 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T13:00:31,399 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T13:00:31,399 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T13:00:31,400 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T13:00:31,400 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T13:00:31,400 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-25T13:00:31,400 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1219616701, stopped=false 2024-11-25T13:00:31,400 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=1ff8009e463e,46289,1732539448369 2024-11-25T13:00:31,443 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35905-0x101735baab60001, quorum=127.0.0.1:60142, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-25T13:00:31,443 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35905-0x101735baab60001, quorum=127.0.0.1:60142, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T13:00:31,443 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T13:00:31,444 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46289-0x101735baab60000, quorum=127.0.0.1:60142, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-25T13:00:31,444 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46289-0x101735baab60000, quorum=127.0.0.1:60142, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T13:00:31,444 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T13:00:31,444 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T13:00:31,444 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T13:00:31,444 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '1ff8009e463e,35905,1732539449246' ***** 2024-11-25T13:00:31,444 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-25T13:00:31,444 INFO [RS:0;1ff8009e463e:35905 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-25T13:00:31,445 INFO [RS:0;1ff8009e463e:35905 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-25T13:00:31,445 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-25T13:00:31,445 INFO [RS:0;1ff8009e463e:35905 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-25T13:00:31,445 INFO [RS:0;1ff8009e463e:35905 {}] regionserver.HRegionServer(959): stopping server 1ff8009e463e,35905,1732539449246 2024-11-25T13:00:31,445 INFO [RS:0;1ff8009e463e:35905 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-25T13:00:31,445 INFO [RS:0;1ff8009e463e:35905 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;1ff8009e463e:35905. 2024-11-25T13:00:31,445 DEBUG [RS:0;1ff8009e463e:35905 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T13:00:31,445 DEBUG [RS:0;1ff8009e463e:35905 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T13:00:31,445 INFO [RS:0;1ff8009e463e:35905 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-25T13:00:31,446 INFO [RS:0;1ff8009e463e:35905 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-25T13:00:31,446 INFO [RS:0;1ff8009e463e:35905 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-25T13:00:31,446 INFO [RS:0;1ff8009e463e:35905 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-25T13:00:31,447 INFO [RS:0;1ff8009e463e:35905 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-25T13:00:31,447 DEBUG [RS:0;1ff8009e463e:35905 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-25T13:00:31,447 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35905-0x101735baab60001, quorum=127.0.0.1:60142, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T13:00:31,447 DEBUG [RS_CLOSE_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-25T13:00:31,447 INFO [RS_CLOSE_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-25T13:00:31,447 DEBUG [RS_CLOSE_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-25T13:00:31,447 DEBUG [RS_CLOSE_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-25T13:00:31,447 DEBUG [RS:0;1ff8009e463e:35905 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-25T13:00:31,447 DEBUG [RS_CLOSE_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-25T13:00:31,447 INFO [RS_CLOSE_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-25T13:00:31,450 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:46289-0x101735baab60000, quorum=127.0.0.1:60142, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T13:00:31,487 INFO [regionserver/1ff8009e463e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T13:00:31,496 DEBUG [RS_CLOSE_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/hbase/meta/1588230740/.tmp/info/66f3c09c9794498587dafcda164d7512 is 95, key is TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85./info:/1732539480958/DeleteFamily/seqid=0 2024-11-25T13:00:31,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742514_1690 (size=6514) 2024-11-25T13:00:31,647 DEBUG [RS:0;1ff8009e463e:35905 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-25T13:00:31,669 INFO [regionserver/1ff8009e463e:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-25T13:00:31,669 INFO [regionserver/1ff8009e463e:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-25T13:00:31,669 INFO [regionserver/1ff8009e463e:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: ExecutorStatusChore was stopped 2024-11-25T13:00:31,848 DEBUG [RS:0;1ff8009e463e:35905 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-25T13:00:31,900 INFO [RS_CLOSE_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/hbase/meta/1588230740/.tmp/info/66f3c09c9794498587dafcda164d7512 2024-11-25T13:00:31,925 DEBUG [RS_CLOSE_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/hbase/meta/1588230740/.tmp/ns/48ef3d4d61534384ab2b7a7ef6ea91e8 is 93, key is TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85./ns:/1732539480883/DeleteFamily/seqid=0 2024-11-25T13:00:31,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742515_1691 (size=5906) 2024-11-25T13:00:32,048 DEBUG [RS:0;1ff8009e463e:35905 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-25T13:00:32,248 DEBUG [RS:0;1ff8009e463e:35905 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-25T13:00:32,328 INFO [RS_CLOSE_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/hbase/meta/1588230740/.tmp/ns/48ef3d4d61534384ab2b7a7ef6ea91e8 2024-11-25T13:00:32,351 DEBUG [RS_CLOSE_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/hbase/meta/1588230740/.tmp/rep_barrier/8c517f1e450e4afdb2743d6bc36b8e0c is 102, key is TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85./rep_barrier:/1732539480883/DeleteFamily/seqid=0 2024-11-25T13:00:32,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742516_1692 (size=6025) 2024-11-25T13:00:32,357 INFO [RS_CLOSE_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/hbase/meta/1588230740/.tmp/rep_barrier/8c517f1e450e4afdb2743d6bc36b8e0c 2024-11-25T13:00:32,381 DEBUG [RS_CLOSE_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/hbase/meta/1588230740/.tmp/table/6cfb25ef33cd465992284bc87c87cfd6 is 96, key is TestAcidGuarantees,,1732539452841.e7fe900c90851bb165c5d4e631eedb85./table:/1732539480883/DeleteFamily/seqid=0 2024-11-25T13:00:32,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742517_1693 (size=5936) 2024-11-25T13:00:32,448 DEBUG [RS:0;1ff8009e463e:35905 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-25T13:00:32,648 DEBUG [RS:0;1ff8009e463e:35905 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-25T13:00:32,790 INFO [RS_CLOSE_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/hbase/meta/1588230740/.tmp/table/6cfb25ef33cd465992284bc87c87cfd6 2024-11-25T13:00:32,794 DEBUG [RS_CLOSE_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/hbase/meta/1588230740/.tmp/info/66f3c09c9794498587dafcda164d7512 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/hbase/meta/1588230740/info/66f3c09c9794498587dafcda164d7512 2024-11-25T13:00:32,798 INFO [RS_CLOSE_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/hbase/meta/1588230740/info/66f3c09c9794498587dafcda164d7512, entries=12, sequenceid=90, filesize=6.4 K 2024-11-25T13:00:32,798 DEBUG [RS_CLOSE_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/hbase/meta/1588230740/.tmp/ns/48ef3d4d61534384ab2b7a7ef6ea91e8 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/hbase/meta/1588230740/ns/48ef3d4d61534384ab2b7a7ef6ea91e8 2024-11-25T13:00:32,801 INFO [RS_CLOSE_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/hbase/meta/1588230740/ns/48ef3d4d61534384ab2b7a7ef6ea91e8, entries=8, sequenceid=90, filesize=5.8 K 2024-11-25T13:00:32,802 DEBUG [RS_CLOSE_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/hbase/meta/1588230740/.tmp/rep_barrier/8c517f1e450e4afdb2743d6bc36b8e0c as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/hbase/meta/1588230740/rep_barrier/8c517f1e450e4afdb2743d6bc36b8e0c 2024-11-25T13:00:32,805 INFO [RS_CLOSE_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/hbase/meta/1588230740/rep_barrier/8c517f1e450e4afdb2743d6bc36b8e0c, entries=6, sequenceid=90, filesize=5.9 K 2024-11-25T13:00:32,805 DEBUG [RS_CLOSE_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/hbase/meta/1588230740/.tmp/table/6cfb25ef33cd465992284bc87c87cfd6 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/hbase/meta/1588230740/table/6cfb25ef33cd465992284bc87c87cfd6 2024-11-25T13:00:32,808 INFO [RS_CLOSE_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/hbase/meta/1588230740/table/6cfb25ef33cd465992284bc87c87cfd6, entries=7, sequenceid=90, filesize=5.8 K 2024-11-25T13:00:32,809 INFO [RS_CLOSE_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1362ms, sequenceid=90, compaction requested=false 2024-11-25T13:00:32,813 DEBUG [RS_CLOSE_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-25T13:00:32,813 DEBUG [RS_CLOSE_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-25T13:00:32,813 INFO [RS_CLOSE_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-25T13:00:32,813 DEBUG [RS_CLOSE_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732539631447Running coprocessor pre-close hooks at 1732539631447Disabling compacts and flushes for region at 1732539631447Disabling writes for close at 1732539631447Obtaining lock to block concurrent updates at 1732539631447Preparing flush snapshotting stores in 1588230740 at 1732539631447Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1732539631448 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732539631450 (+2 ms)Flushing 1588230740/info: creating writer at 1732539631450Flushing 1588230740/info: appending metadata at 1732539631495 (+45 ms)Flushing 1588230740/info: closing flushed file at 1732539631495Flushing 1588230740/ns: creating writer at 1732539631906 (+411 ms)Flushing 1588230740/ns: appending metadata at 1732539631925 (+19 ms)Flushing 1588230740/ns: closing flushed file at 1732539631925Flushing 1588230740/rep_barrier: creating writer at 1732539632331 (+406 ms)Flushing 1588230740/rep_barrier: appending metadata at 1732539632350 (+19 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1732539632350Flushing 1588230740/table: creating writer at 1732539632361 (+11 ms)Flushing 1588230740/table: appending metadata at 1732539632380 (+19 ms)Flushing 1588230740/table: closing flushed file at 1732539632380Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@294d9ee: reopening flushed file at 1732539632794 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@39e34483: reopening flushed file at 1732539632798 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@28bb7152: reopening flushed file at 1732539632801 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7eade074: reopening flushed file at 1732539632805 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1362ms, sequenceid=90, compaction requested=false at 1732539632809 (+4 ms)Writing region close event to WAL at 1732539632811 (+2 ms)Running coprocessor post-close hooks at 1732539632813 (+2 ms)Closed at 1732539632813 2024-11-25T13:00:32,814 DEBUG [RS_CLOSE_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-25T13:00:32,849 INFO [RS:0;1ff8009e463e:35905 {}] regionserver.HRegionServer(976): stopping server 1ff8009e463e,35905,1732539449246; all regions closed. 2024-11-25T13:00:32,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741834_1010 (size=25344) 2024-11-25T13:00:32,853 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/WALs/1ff8009e463e,35905,1732539449246/1ff8009e463e%2C35905%2C1732539449246.meta.1732539452300.meta not finished, retry = 0 2024-11-25T13:00:32,956 DEBUG [RS:0;1ff8009e463e:35905 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/oldWALs 2024-11-25T13:00:32,956 INFO [RS:0;1ff8009e463e:35905 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 1ff8009e463e%2C35905%2C1732539449246.meta:.meta(num 1732539452300) 2024-11-25T13:00:32,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741833_1009 (size=15592245) 2024-11-25T13:00:32,960 DEBUG [RS:0;1ff8009e463e:35905 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/oldWALs 2024-11-25T13:00:32,960 INFO [RS:0;1ff8009e463e:35905 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 1ff8009e463e%2C35905%2C1732539449246:(num 1732539451791) 2024-11-25T13:00:32,960 DEBUG [RS:0;1ff8009e463e:35905 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T13:00:32,960 INFO [RS:0;1ff8009e463e:35905 {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T13:00:32,960 INFO [RS:0;1ff8009e463e:35905 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-25T13:00:32,960 INFO [RS:0;1ff8009e463e:35905 {}] hbase.ChoreService(370): Chore service for: regionserver/1ff8009e463e:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-25T13:00:32,961 INFO [RS:0;1ff8009e463e:35905 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-25T13:00:32,961 INFO [regionserver/1ff8009e463e:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-25T13:00:32,961 INFO [RS:0;1ff8009e463e:35905 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:35905 2024-11-25T13:00:32,976 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46289-0x101735baab60000, quorum=127.0.0.1:60142, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-25T13:00:32,976 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35905-0x101735baab60001, quorum=127.0.0.1:60142, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/1ff8009e463e,35905,1732539449246 2024-11-25T13:00:32,976 INFO [RS:0;1ff8009e463e:35905 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-25T13:00:32,977 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [1ff8009e463e,35905,1732539449246] 2024-11-25T13:00:32,985 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/1ff8009e463e,35905,1732539449246 already deleted, retry=false 2024-11-25T13:00:32,985 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 1ff8009e463e,35905,1732539449246 expired; onlineServers=0 2024-11-25T13:00:32,985 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '1ff8009e463e,46289,1732539448369' ***** 2024-11-25T13:00:32,985 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-25T13:00:32,985 INFO [M:0;1ff8009e463e:46289 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-25T13:00:32,985 INFO [M:0;1ff8009e463e:46289 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-25T13:00:32,985 DEBUG [M:0;1ff8009e463e:46289 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-25T13:00:32,985 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-25T13:00:32,985 DEBUG [M:0;1ff8009e463e:46289 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-25T13:00:32,985 DEBUG [master/1ff8009e463e:0:becomeActiveMaster-HFileCleaner.large.0-1732539451229 {}] cleaner.HFileCleaner(306): Exit Thread[master/1ff8009e463e:0:becomeActiveMaster-HFileCleaner.large.0-1732539451229,5,FailOnTimeoutGroup] 2024-11-25T13:00:32,986 INFO [M:0;1ff8009e463e:46289 {}] hbase.ChoreService(370): Chore service for: master/1ff8009e463e:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-25T13:00:32,986 DEBUG [master/1ff8009e463e:0:becomeActiveMaster-HFileCleaner.small.0-1732539451256 {}] cleaner.HFileCleaner(306): Exit Thread[master/1ff8009e463e:0:becomeActiveMaster-HFileCleaner.small.0-1732539451256,5,FailOnTimeoutGroup] 2024-11-25T13:00:32,986 INFO [M:0;1ff8009e463e:46289 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-25T13:00:32,986 DEBUG [M:0;1ff8009e463e:46289 {}] master.HMaster(1795): Stopping service threads 2024-11-25T13:00:32,986 INFO [M:0;1ff8009e463e:46289 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-25T13:00:32,986 INFO [M:0;1ff8009e463e:46289 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T13:00:32,986 ERROR [M:0;1ff8009e463e:46289 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-25T13:00:32,987 INFO [M:0;1ff8009e463e:46289 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-25T13:00:32,987 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-25T13:00:32,993 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46289-0x101735baab60000, quorum=127.0.0.1:60142, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-25T13:00:32,993 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46289-0x101735baab60000, quorum=127.0.0.1:60142, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T13:00:32,993 DEBUG [M:0;1ff8009e463e:46289 {}] zookeeper.ZKUtil(347): master:46289-0x101735baab60000, quorum=127.0.0.1:60142, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-25T13:00:32,993 WARN [M:0;1ff8009e463e:46289 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-25T13:00:32,994 INFO [M:0;1ff8009e463e:46289 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/.lastflushedseqids 2024-11-25T13:00:33,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742518_1694 (size=72) 2024-11-25T13:00:33,085 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35905-0x101735baab60001, quorum=127.0.0.1:60142, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T13:00:33,085 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35905-0x101735baab60001, quorum=127.0.0.1:60142, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T13:00:33,085 INFO [RS:0;1ff8009e463e:35905 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-25T13:00:33,086 INFO [RS:0;1ff8009e463e:35905 {}] regionserver.HRegionServer(1031): Exiting; stopping=1ff8009e463e,35905,1732539449246; zookeeper connection closed. 2024-11-25T13:00:33,086 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5a0ad95d {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5a0ad95d 2024-11-25T13:00:33,086 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-25T13:00:33,405 INFO [M:0;1ff8009e463e:46289 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-25T13:00:33,405 INFO [M:0;1ff8009e463e:46289 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-25T13:00:33,405 DEBUG [M:0;1ff8009e463e:46289 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-25T13:00:33,405 INFO [M:0;1ff8009e463e:46289 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T13:00:33,405 DEBUG [M:0;1ff8009e463e:46289 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T13:00:33,405 DEBUG [M:0;1ff8009e463e:46289 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-25T13:00:33,405 DEBUG [M:0;1ff8009e463e:46289 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T13:00:33,405 INFO [M:0;1ff8009e463e:46289 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=743.40 KB heapSize=913.55 KB 2024-11-25T13:00:33,423 DEBUG [M:0;1ff8009e463e:46289 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1d1ff26a6a8f424ea5916d1fc220bcbf is 82, key is hbase:meta,,1/info:regioninfo/1732539452414/Put/seqid=0 2024-11-25T13:00:33,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742519_1695 (size=5672) 2024-11-25T13:00:33,827 INFO [M:0;1ff8009e463e:46289 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2101 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1d1ff26a6a8f424ea5916d1fc220bcbf 2024-11-25T13:00:33,855 DEBUG [M:0;1ff8009e463e:46289 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/8a95d3e209e34befb33eb654d9d0fcb2 is 2275, key is \x00\x00\x00\x00\x00\x00\x00\x87/proc:d/1732539601397/Put/seqid=0 2024-11-25T13:00:33,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742520_1696 (size=43071) 2024-11-25T13:00:33,860 INFO [M:0;1ff8009e463e:46289 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=742.84 KB at sequenceid=2101 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/8a95d3e209e34befb33eb654d9d0fcb2 2024-11-25T13:00:33,877 INFO [M:0;1ff8009e463e:46289 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 8a95d3e209e34befb33eb654d9d0fcb2 2024-11-25T13:00:33,900 DEBUG [M:0;1ff8009e463e:46289 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/63251d6a71eb46b9a2c4f5160c316ce0 is 69, key is 1ff8009e463e,35905,1732539449246/rs:state/1732539451371/Put/seqid=0 2024-11-25T13:00:33,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073742521_1697 (size=5156) 2024-11-25T13:00:34,305 INFO [M:0;1ff8009e463e:46289 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2101 (bloomFilter=true), to=hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/63251d6a71eb46b9a2c4f5160c316ce0 2024-11-25T13:00:34,309 DEBUG [M:0;1ff8009e463e:46289 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1d1ff26a6a8f424ea5916d1fc220bcbf as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/1d1ff26a6a8f424ea5916d1fc220bcbf 2024-11-25T13:00:34,311 INFO [M:0;1ff8009e463e:46289 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/1d1ff26a6a8f424ea5916d1fc220bcbf, entries=8, sequenceid=2101, filesize=5.5 K 2024-11-25T13:00:34,312 DEBUG [M:0;1ff8009e463e:46289 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/8a95d3e209e34befb33eb654d9d0fcb2 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/8a95d3e209e34befb33eb654d9d0fcb2 2024-11-25T13:00:34,315 INFO [M:0;1ff8009e463e:46289 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 8a95d3e209e34befb33eb654d9d0fcb2 2024-11-25T13:00:34,315 INFO [M:0;1ff8009e463e:46289 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/8a95d3e209e34befb33eb654d9d0fcb2, entries=164, sequenceid=2101, filesize=42.1 K 2024-11-25T13:00:34,315 DEBUG [M:0;1ff8009e463e:46289 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/63251d6a71eb46b9a2c4f5160c316ce0 as hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/63251d6a71eb46b9a2c4f5160c316ce0 2024-11-25T13:00:34,319 INFO [M:0;1ff8009e463e:46289 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32849/user/jenkins/test-data/cdc703a9-0daa-e6b0-295d-59eb18c5ac7a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/63251d6a71eb46b9a2c4f5160c316ce0, entries=1, sequenceid=2101, filesize=5.0 K 2024-11-25T13:00:34,319 INFO [M:0;1ff8009e463e:46289 {}] regionserver.HRegion(3140): Finished flush of dataSize ~743.40 KB/761237, heapSize ~913.25 KB/935168, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 914ms, sequenceid=2101, compaction requested=false 2024-11-25T13:00:34,323 INFO [M:0;1ff8009e463e:46289 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T13:00:34,323 DEBUG [M:0;1ff8009e463e:46289 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732539633405Disabling compacts and flushes for region at 1732539633405Disabling writes for close at 1732539633405Obtaining lock to block concurrent updates at 1732539633405Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732539633405Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=761237, getHeapSize=935408, getOffHeapSize=0, getCellsCount=2290 at 1732539633406 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732539633406Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732539633406Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732539633423 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732539633423Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732539633831 (+408 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732539633855 (+24 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732539633855Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732539633877 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732539633900 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732539633900Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@79b2414d: reopening flushed file at 1732539634308 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@289ea192: reopening flushed file at 1732539634311 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@558f552e: reopening flushed file at 1732539634315 (+4 ms)Finished flush of dataSize ~743.40 KB/761237, heapSize ~913.25 KB/935168, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 914ms, sequenceid=2101, compaction requested=false at 1732539634320 (+5 ms)Writing region close event to WAL at 1732539634323 (+3 ms)Closed at 1732539634323 2024-11-25T13:00:34,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38683 is added to blk_1073741830_1006 (size=898977) 2024-11-25T13:00:34,327 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-25T13:00:34,327 INFO [M:0;1ff8009e463e:46289 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-25T13:00:34,327 INFO [M:0;1ff8009e463e:46289 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:46289 2024-11-25T13:00:34,327 INFO [M:0;1ff8009e463e:46289 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-25T13:00:34,518 INFO [M:0;1ff8009e463e:46289 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-25T13:00:34,518 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46289-0x101735baab60000, quorum=127.0.0.1:60142, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T13:00:34,518 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46289-0x101735baab60000, quorum=127.0.0.1:60142, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T13:00:34,523 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7acb84b5{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-25T13:00:34,526 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6e1ebae6{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-25T13:00:34,526 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-25T13:00:34,526 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-25T13:00:34,526 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@42946cd0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2bb80d27-3d57-c947-53d6-c4472333a06b/hadoop.log.dir/,STOPPED} 2024-11-25T13:00:34,532 WARN [BP-2055677215-172.17.0.3-1732539443838 heartbeating to localhost/127.0.0.1:32849 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-25T13:00:34,532 WARN [BP-2055677215-172.17.0.3-1732539443838 heartbeating to localhost/127.0.0.1:32849 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2055677215-172.17.0.3-1732539443838 (Datanode Uuid 5b24ef01-cddf-4ceb-adbc-7bc07f6c930d) service to localhost/127.0.0.1:32849 2024-11-25T13:00:34,534 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-25T13:00:34,534 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-25T13:00:34,536 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-25T13:00:34,536 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2bb80d27-3d57-c947-53d6-c4472333a06b/cluster_8b8d8747-9aba-2792-61f4-67cb1dbecf7b/data/data1/current/BP-2055677215-172.17.0.3-1732539443838 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-25T13:00:34,537 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2bb80d27-3d57-c947-53d6-c4472333a06b/cluster_8b8d8747-9aba-2792-61f4-67cb1dbecf7b/data/data2/current/BP-2055677215-172.17.0.3-1732539443838 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-25T13:00:34,553 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-25T13:00:34,554 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-25T13:00:34,554 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-25T13:00:34,554 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-25T13:00:34,554 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2bb80d27-3d57-c947-53d6-c4472333a06b/hadoop.log.dir/,STOPPED} 2024-11-25T13:00:34,575 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-25T13:00:34,742 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down